Alysha old pre rebase

.travis.yml

@@ -15,21 +15,19 @@ env:
   global:
     - PATH=$HOME/cached-deps:$PATH
   jobs:
-    # Most recent releases of the last two minor releases of Pachyderm
-    - PACHYDERM_VERSION=1.12.5
-    - PACHYDERM_VERSION=1.13.0
+    - PACHYDERM_VERSION=2.0.0-alpha.12
 
 jobs:
   include:
     # other tests
     - python: 3.8
       env:
         - TOXENV=lint
-        - PACHYDERM_VERSION=1.13.0
+        - PACHYDERM_VERSION=2.0.0-alpha.12
     - python: 3.8
       env:
         - TOXENV=examples
-        - PACHYDERM_VERSION=1.13.0
+        - PACHYDERM_VERSION=2.0.0-alpha.12
 install:
 - make ci-install
 before_script:

Makefile

@@ -19,15 +19,17 @@ docker-build-proto:
 
 src/python_pachyderm/proto: docker-build-proto
 	@echo "Building with pachyderm core v$(PACHYDERM_VERSION)"
+	rm -rf src/python_pachyderm/proto
 	cd proto/pachyderm && \
 		git fetch --all && \
 		git checkout v$(PACHYDERM_VERSION)
-	find ./proto/pachyderm/src/client -regex ".*\.proto" \
+	find ./proto/pachyderm/ -regex ".*\.proto" \
+	| grep -v 'internal' \
 	| xargs tar cf - \
 	| docker run -i pachyderm_python_proto \
 	| tar xf -
-	test -d src/python_pachyderm/proto && rm -rf src/python_pachyderm/proto
-	mv src/python_pachyderm/client src/python_pachyderm/proto
+	rm -rf src/python_pachyderm/proto
+	mv src/python_pachyderm/src src/python_pachyderm/proto
 	find src/python_pachyderm/proto -type d -exec touch {}/__init__.py \;
 
 init:
@@ -36,23 +38,22 @@ init:
 ci-install:
 	sudo apt-get update
 	sudo apt-get install jq socat
-	cd proto/pachyderm && \
-		sudo etc/testing/travis_cache.sh && \
-		sudo etc/testing/travis_install.sh && \
-		curl -o /tmp/pachctl.deb -L https://github.com/pachyderm/pachyderm/releases/download/v$(PACHYDERM_VERSION)/pachctl_$(PACHYDERM_VERSION)_amd64.deb  && \
-		sudo dpkg -i /tmp/pachctl.deb
+	sudo etc/testing/travis_cache.sh
+	sudo etc/testing/travis_install.sh
+	curl -o /tmp/pachctl.deb -L https://github.com/pachyderm/pachyderm/releases/download/v$(PACHYDERM_VERSION)/pachctl_$(PACHYDERM_VERSION)_amd64.deb
+	sudo dpkg -i /tmp/pachctl.deb
 	pip install tox tox-travis
 
-ci-setup:
+ci-setup: 
 	docker version
 	which pachctl
-	cd proto/pachyderm && make launch-kube
-	pachctl deploy local
-	until timeout 1s ./proto/pachyderm/etc/kube/check_ready.sh app=pachd; do sleep 1; done
+	etc/kube/start-minikube.sh
+	echo 'y' | pachctl deploy local
+	until timeout 1s ./etc/kube/check_ready.sh app=pachd; do sleep 1; done
 	pachctl version
 
 release:
-	git checkout master
+	git checkout v7.x
 	rm -rf build dist
 	python3 setup.py sdist
 	twine upload dist/*
@@ -61,4 +62,4 @@ lint:
 	flake8 src/python_pachyderm --exclude=src/python_pachyderm/proto --max-line-length=120 --max-doc-length=80
 	PYTHONPATH=./src:$(PYTHONPATH) etc/proto_lint/proto_lint.py
 
-.PHONY: docker-build-proto init ci-install ci-setup release lint
+.PHONY: docs docker-build-proto init ci-install ci-setup release lint

etc/kube/check_ready.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+if [ -z "$1" ]; then
+  echo "Usage $0 label=value <optional_namespace>"
+  exit 1
+fi
+
+namespace=default
+
+if [ -n "$2" ]; then
+    namespace=$2
+fi
+
+results=$(kubectl get pods \
+  -l "$1" \
+  --namespace="$namespace" \
+  -o jsonpath='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' \
+   | tr ';' "\n")
+
+if [ -z "$results" ]; then
+  echo "Empty result"
+  echo "$results"
+  exit 1
+fi
+
+
+readyPods=$(echo "$results" | tr ' ' "\n" | grep -c "Ready=True")
+allPods=$(echo "$results" | tr ' ' "\n" | grep -c "Ready=")
+
+if [ "$allPods" -eq 0 ]; then
+    echo "No pods found yet"
+    exit 1
+fi
+
+if [ "$readyPods" -ne "$allPods" ]; then
+    echo "${readyPods}/${allPods} ready"
+    exit 1
+fi
+
+echo "All pods are ready."
+exit 0

etc/kube/internal.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -Ee
+
+jq '.spec.containers[1].command |= . + [ "--storage-backend=etcd2" ]' </etc/kubernetes/manifests/master.json >/etc/kubernetes/manifests/master2.json
+mv /etc/kubernetes/manifests/master2.json /etc/kubernetes/manifests/master.json
+/hyperkube kubelet \
+    --containerized \
+    --hostname-override="127.0.0.1" \
+    --address="0.0.0.0" \
+    --api-servers=http://localhost:8080 \
+    --cluster_dns=10.0.0.10 \
+    --cluster_domain=cluster.local \
+    --pod-manifest-path=/etc/kubernetes/manifests \
+    --allow-privileged=true \
+    --feature-gates="Accelerators=true"

etc/kube/start-minikube.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -Eex
+
+# Parse flags
+VERSION=v1.13.0
+minikube_args=(
+  "--vm-driver=none"
+  "--kubernetes-version=${VERSION}"
+)
+while getopts ":v" opt; do
+  case "${opt}" in
+    v)
+      VERSION="v${OPTARG}"
+      ;;
+    \?)
+      echo "Invalid argument: ${opt}"
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -n "${TRAVIS}" ]]; then
+  minikube_args+=("--bootstrapper=kubeadm")
+fi
+
+
+# Note that we update the 'PATH' to include '~/cached-deps' in '.travis.yml',
+# but this doesn't update the PATH for calls using 'sudo'. To make a 'sudo' run
+# a binary in '~/cached-deps', you need to explicitly set the path like so:
+#     sudo env "PATH=$PATH" minikube foo
+sudo env "PATH=$PATH" "CHANGE_MINIKUBE_NONE_USER=true" \
+  minikube start "${minikube_args[@]}"
+
+# Try to connect for three minutes
+for _ in $(seq 36); do
+  if kubectl version &>/dev/null; then
+    exit 0
+  fi
+  sleep 5
+done
+
+# Give up--kubernetes isn't coming up
+minikube delete
+sleep 30 # Wait for minikube to go completely down
+exit 1

etc/proto_lint/proto_lint.py

@@ -23,6 +23,8 @@
     Service.DEBUG: mixin.debug.DebugMixin,
     Service.ENTERPRISE: mixin.enterprise.EnterpriseMixin,
     Service.HEALTH: mixin.health.HealthMixin,
+    Service.IDENTITY: mixin.identity.IdentityMixin,
+    Service.LICENSE: mixin.license.LicenseMixin,
     Service.PFS: mixin.pfs.PFSMixin,
     Service.PPS: mixin.pps.PPSMixin,
     Service.TRANSACTION: mixin.transaction.TransactionMixin,
@@ -66,23 +68,34 @@
         "build_commit",
         "put_tar",
         "get_tar",
-        # ignore storage v2 for now
-        "get_tar_conditional_v2",
-        "file_operation_v2",
-        "list_file_v2",
-        "glob_file_v2",
-        "get_tar_v2",
-        "walk_file_v2",
-        "inspect_file_v2",
-        "clear_commit_v2",
-        "diff_file_v2",
+        "activate_auth",
+        # TODO: add these new API methods
+        "renew_fileset",
+        "squash_commit",
+        "clear_commit",
+        "modify_file",
+        "add_fileset",
+        "get_fileset",
+        "create_fileset",
     ],
     Service.PPS: [
         # the following are ignored because they're for internal use only
         "activate_auth",
         "create_job",
         "update_job_state",
     ],
+    Service.AUTH: [
+        "get_auth_token",
+        "extend_auth_token",
+    ],
+    Service.ENTERPRISE: [
+        # internal RPC only
+        "heartbeat",
+    ],
+    Service.LICENSE: [
+        # internal RPC only
+        "heartbeat",
+    ],
 }
 
 # Mapping of what the linter would by default expect a proto name to be, to
@@ -91,7 +104,7 @@
     Service.AUTH: {
         "activate": ["activate_auth"],
         "deactivate": ["deactivate_auth"],
-        "authenticate": ["authenticate_github", "authenticate_oidc", "authenticate_one_time_password"],
+        "authenticate": ["authenticate_oidc"],
         "get_a_c_l": ["get_acl"],
         "set_a_c_l": ["set_acl"],
         "get_o_i_d_c_login": ["get_oidc_login"],
@@ -106,6 +119,23 @@
         "get_state": ["get_enterprise_state"],
         "deactivate": ["deactivate_enterprise"],
     },
+    Service.LICENSE: {
+        "activate": ["activate_license"],
+        "delete_all": ["delete_all_license"],
+    },
+    Service.IDENTITY: {
+        "get_i_d_p_connector": ["get_idp_connector"],
+        "delete_i_d_p_connector": ["delete_idp_connector"],
+        "list_i_d_p_connectors": ["list_idp_connectors"],
+        "create_i_d_p_connector": ["create_idp_connector"],
+        "update_i_d_p_connector": ["update_idp_connector"],
+	"get_o_i_d_c_client": ["get_oidc_client"],
+        "delete_o_i_d_c_client": ["delete_oidc_client"],
+        "list_o_i_d_c_clients": ["list_oidc_clients"],
+        "create_o_i_d_c_client": ["create_oidc_client"],
+        "update_o_i_d_c_client": ["update_oidc_client"],
+        "delete_all": ["delete_all_identity"],
+    },
     Service.PFS: {
         "put_file": ["put_file_bytes", "put_file_url"],
     },
@@ -132,27 +162,7 @@
 # * We can also specify that a method has an argument that isn't in the
 #   protos: `(None, "ignored_arg_name")`
 RENAMED_ARGS = {
-    # admin
-    "extract_pipeline": [
-        ("pipeline", "pipeline_name"),
-    ],
-    "extract": [
-        ("URL", "url"),
-    ],
-    "restore": [
-        (("op", "URL"), "requests"),
-    ],
     # auth
-    "authenticate_github": [
-        ("one_time_password", None),
-        ("oidc_state", None),
-        ("id_token", None),
-    ],
-    "authenticate_one_time_password": [
-        ("github_token", None),
-        ("oidc_state", None),
-        ("id_token", None),
-    ],
     "authenticate_oidc": [
         ("github_token", None),
         ("one_time_password", None),

etc/testing/kube_debug.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+echo "=== TEST FAILED OR TIMED OUT, DUMPING DEBUG INFO ==="
+
+# TODO: Extend this to show kubectl describe output for failed pods, this will
+# probably show why things are hanging.
+
+# SC2016 blocks variables in single-quoted strings, but these are 'eval'ed below
+# shellcheck disable=SC2016
+cmds=(
+  'pachctl version'
+  'pachctl list repo'
+  'pachctl list repo --raw | jq -r ".repo.name" | while read r; do
+     echo "---";
+     pachctl inspect repo "${r}";
+     echo "Commits:";
+     pachctl list commit "${r}";
+     echo;
+   done'
+  'pachctl list pipeline'
+  'pachctl list job'
+  'kubectl version'
+  'kubectl get all --all-namespaces'
+  'kubectl describe pod -l suite=pachyderm,app=pachd'
+  'kubectl describe pod -l suite=pachyderm,app=etcd'
+  # Set --tail b/c by default 'kubectl logs' only outputs 10 lines if -l is set
+  'kubectl logs --tail=500 -l suite=pachyderm,app=pachd'
+  'kubectl logs --tail=1000 -l suite=pachyderm,app=pachd --previous # if pachd restarted'
+  'sudo dmesg | tail -n 40'
+  'minikube logs | tail -n 100'
+  'top -b -n 1 | head -n 40'
+  'df -h'
+)
+for c in "${cmds[@]}"; do
+  echo "======================================================================"
+  echo "${c}"
+  echo "----------------------------------------------------------------------"
+  eval "${c}"
+done
+echo "======================================================================"

etc/testing/minikube_version

@@ -0,0 +1 @@
+1.13.1

etc/testing/travis_cache.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Create all of our Travis cached dirs, or, if they exist, loosen their
+# permissions enough that our build process is guaranteed to be able to write
+# to them and Travis's caching process is guaranteed to be able to read them.
+# (Travis caching requires that all cached files be readable, and, I surmise,
+# that cached directories be executable, to be traversed).
+
+set -ex
+
+# If any directories are added or changed here, they must also be added to
+# .travis.yml
+dirs=(
+  "${HOME}/.cache"
+  "${HOME}/cached-deps"
+  "${GOPATH}/pkg"
+  "$(python3 -c 'import site; print(site.USER_BASE)')" # $HOME/.local
+)
+
+for dir in "${dirs[@]}"; do
+  if [[ -d "${dir}" ]]; then
+    # change ownership on directory, in case it's owned by root
+    sudo chown -R "${USER}:${USER}" "${dir}"
+  else
+    # create directory
+    mkdir -p "${dir}"
+  fi
+  # Loosen permissions so build processes can write and Travis can cache
+  chmod 777 -R "${dir}"
+done