Centralize Makefile logic in baselayer (#229)

* Remove unneeded config/print

* Centralize Makefile logic in baselayer

* Rename config.yaml.example to config.yaml.defaults

* Move test Makefile target to baselayer

* Always initialize submodule first for any make target

Author: bnaul

.gitignore

@@ -13,6 +13,5 @@ ghostdriver.log
 __pycache__/
 node_modules/
 *.pyc
-_test_config.yaml
 config.yaml
 data

.travis/travis_install.sh

@@ -17,8 +17,6 @@ section "install.cesium_web.requirements"
 npm -g install npm@latest
 npm --version
 node --version
-make dependencies
-make check-js-updates
 
 if [[ ${TRIGGERED_FROM_REPO} == "cesium-ml/cesium" ]]; then
     mkdir cesium-clone
@@ -35,11 +33,9 @@ pip list --format=columns
 section_end "install.cesium_web.requirements"
 
 
-section "init.cesium_web"
-make paths
+section "init.db"
 make db_init
-make bundle
-section_end "init.cesium_web"
+section_end "init.db"
 
 
 section "install.chromedriver"

Makefile

@@ -1,107 +1,39 @@
-SHELL = /bin/bash
-SUPERVISORD=FLAGS=$$FLAGS supervisord -c baselayer/conf/supervisor/supervisor.conf
-SUPERVISORCTL=FLAGS=$$FLAGS supervisorctl -c baselayer/conf/supervisor/supervisor.conf
-ENV_SUMMARY=PYTHONPATH=. ./baselayer/tools/env_summary.py $$FLAGS
-ESLINT=./node_modules/.bin/eslint
-
 .DEFAULT_GOAL := run
 
-bundle = ./static/build/bundle.js
-webpack = ./node_modules/.bin/webpack
-baselayer_branch = $(shell git config -f .gitmodules submodule.baselayer.branch)
-
 baselayer/README.md:
 	git submodule update --init --remote
 	$(MAKE) baselayer-update
 
-.PHONY: baselayer-update
+.PHONY: baselayer-update run log
 baselayer-update:
 	./baselayer/tools/submodule_update.sh
 
-dependencies: baselayer/README.md
-	@./baselayer/tools/silent_monitor.py pip install -r baselayer/requirements.txt
-	@./baselayer/tools/silent_monitor.py pip install -r requirements.txt
-	@./baselayer/tools/silent_monitor.py ./baselayer/tools/check_js_deps.sh
-
-db_init:
-	@PYTHONPATH=. ./baselayer/tools/silent_monitor.py ./baselayer/tools/db_init.py
-
-db_clear:
-	PYTHONPATH=. ./baselayer/tools/db_init.py -f
-
-$(bundle): webpack.config.js package.json
-	$(webpack)
-
-bundle: $(bundle)
-
-bundle-watch:
-	$(webpack) -w
+log:
+	make -C baselayer log
 
-paths:
-	@mkdir -p log run tmp
-	@mkdir -p log/sv_child
-	@mkdir -p ~/.local/cesium/logs
+run:
+	make -C baselayer run
 
-fill_conf_values:
-	PYTHONPATH=. ./baselayer/tools/fill_conf_values.py
-
-log: paths
-	./baselayer/tools/watch_logs.py
-
-run: paths dependencies fill_conf_values
-	@echo "Supervisor will now fire up various micro-services."
-	@echo
-	@echo " - Run \`make log\` in another terminal to view logs"
-	@echo " - Run \`make monitor\` in another terminal to restart services"
-	@echo
-	@echo "The server is in debug mode:"
-	@echo "  JavaScript and Python files will be reloaded upon change."
-	@echo
-
-	@FLAGS="--debug" && \
-	$(ENV_SUMMARY) && echo && \
-	echo "Press Ctrl-C to abort the server" && \
-	echo && \
-	$(SUPERVISORD)
+run_testing:
+	make -C baselayer run_testing
 
 run_production:
-	export FLAGS="--config config.yaml" && \
-	$(ENV_SUMMARY) && \
-	$(SUPERVISORD)
-
-run_testing: paths dependencies
-	export FLAGS="--config _test_config.yaml" && \
-	$(ENV_SUMMARY) && \
-	$(SUPERVISORD)
-
-monitor:
-	@echo "Entering supervisor control panel."
-	@echo " - Type \`status\` too see microservice status"
-	$(SUPERVISORCTL) -i status
+	make -C baselayer run_production
 
-# Attach to terminal of running webserver; useful to, e.g., use pdb
-attach:
-	$(SUPERVISORCTL) fg app
+test:
+	make -C baselayer test
 
-clean:
-	rm $(bundle)
+test_headless:
+	make -C baselayer test_headless
 
-test_headless: paths dependencies fill_conf_values
-	PYTHONPATH='.' xvfb-run ./tools/test_frontend.py
-
-test: paths dependencies fill_conf_values
-	PYTHONPATH='.' ./tools/test_frontend.py
-
-stop:
-	$(SUPERVISORCTL) stop all
+db_init:
+	make -C baselayer db_init
 
-status:
-	PYTHONPATH='.' ./baselayer/tools/supervisor_status.py
+db_clear:
+	make -C baselayer db_clear
 
 docker-images:
 	# Add --no-cache flag to rebuild from scratch
 	docker build -t cesium/web . && docker push cesium/web
 
-# Call this target to see which Javascript dependencies are not up to date
-check-js-updates:
-	./baselayer/tools/check_js_updates.sh
+-include "baselayer/README.md"  # always clone baselayer if it doesn't exist

README.md

@@ -54,8 +54,7 @@ The easiest way to try the web app is to run it through Docker:
     ```
     and restart `postgresl` (`sudo service postgresql restart`).
 
-2. Install Python and JavaScript dependencies with `make_dependencies`
-3. Initialize the database with `make db_init`
+2. Initialize the database with `make db_init`
 
 If you've run this script before, you may see warnings here about the
 database already existing.  Ignore those.
@@ -64,7 +63,7 @@ database already existing.  Ignore those.
 
 ## Configuration
 
-Copy `config.yaml.example` to `config.yaml` and customize.
+Copy `config.yaml.defaults` to `config.yaml` and customize.
 
 - Under `server`, set 'multi_user' to True to enable logins.  Also provide the
   Google credentials, obtained as described in the config file.

baselayer

@@ -22,9 +22,10 @@ def post(self):
             return self.error('No tar file uploaded')
 
         zipfile = data['tarFile']
-        tarball_content_type_str = 'data:application/gzip;base64,'
+        tarball_content_types = ('data:application/gzip;base64',
+                                 'data:application/x-gzip;base64')
 
-        if not zipfile['body'].startswith(tarball_content_type_str):
+        if not zipfile['body'].startswith(tarball_content_types):
             return self.error('Invalid tar file - please ensure file is gzip '
                               'format.')
 
@@ -38,9 +39,10 @@ def post(self):
                         util.secure_filename(zipfile['name']))
         zipfile_path = pjoin(self.cfg['paths:upload_folder'], zipfile_name)
 
+        for prefix in tarball_content_types:
+            zipfile['body'] = zipfile['body'].replace(prefix, '')
         with open(zipfile_path, 'wb') as f:
-            f.write(base64.b64decode(
-                zipfile['body'].replace(tarball_content_type_str, '')))
+            f.write(base64.b64decode(zipfile['body']))
         try:
             tarfile.open(zipfile_path)
         except tarfile.ReadError:

cesium_app/handlers/dataset.py

@@ -13,7 +13,7 @@
 from selenium.common.exceptions import TimeoutException
 from seleniumrequests.request import RequestMixin
 from pytest_factoryboy import register, LazyFixture
-from baselayer.app.config import Config
+from baselayer.app.config import load_config
 from baselayer.app.test_util import (driver, MyCustomWebDriver, reset_state,
                                      set_server_url)
 from cesium_app import models
@@ -22,10 +22,10 @@
                                        PredictionFactory)
 
 
-print('Loading test configuration from _test_config.yaml')
+print('Loading test configuration from test_config.yaml')
 basedir = pathlib.Path(os.path.dirname(__file__))/'../..'
-cfg = Config([basedir/'config.yaml.example', basedir/'_test_config.yaml'])
-set_server_url(cfg['server:url'])
+cfg = load_config([basedir/'test_config.yaml'])
+set_server_url(f'http://localhost:{cfg["ports:app"]}')
 print('Setting test database to:', cfg['database'])
 models.init_db(**cfg['database'])
 

cesium_app/tests/conftest.py

@@ -234,7 +234,6 @@ def test_predict_specific_ts_name(driver, project, dataset, featureset, model):
     response = driver.request(
         'POST', '{}/predictions'.format(driver.server_url),
         json=data).json()
-    print(response)
     assert response['status'] == 'success'
 
     for i in range(10):

cesium_app/tests/frontend/test_predict.py

@@ -30,8 +30,6 @@ database:
     password:
 
 server:
-    url: http://localhost:5000
-
     # From https://console.developers.google.com/
     #
     # - Create Client ID
@@ -44,13 +42,3 @@ server:
         debug_login: True
         google_oauth2_key:
         google_oauth2_secret:
-
-ports:
-    websocket: 64000
-    fake_oauth: 63000
-    app: 5000
-    app_http_proxy: 5001
-    app_internal: 65000
-    dask: 63500
-    websocket_path_in: 'ipc:///tmp/message_flow_in'
-    websocket_path_out: 'ipc:///tmp/message_flow_out'

config.yaml.example renamed to config.yaml.defaults

@@ -0,0 +1,11 @@
+database:
+  database: cesium_test
+  user: cesium
+  host: localhost
+  port: 5432
+
+server:
+  auth:
+    debug_login: True
+    google_oauth2_key:
+    google_oauth2_secret: