Fill config file templates with port values from yaml file (#228)

* Parse template config files before running app; add ports section to config.yaml

* Return file names in response after creating new dataset

* Add generate_token script and use it to fix test_predict_specific_ts_name

* Update Makefile

Author: acrellin

Makefile

@@ -42,11 +42,14 @@ paths:
 	@mkdir -p log/sv_child
 	@mkdir -p ~/.local/cesium/logs
 
+fill_conf_values:
+	PYTHONPATH=. ./baselayer/tools/fill_conf_values.py
+
 log: paths
 	./baselayer/tools/watch_logs.py
 
-run: paths dependencies
-	@echo "Supervisor will now fire up various services."
+run: paths dependencies fill_conf_values
+	@echo "Supervisor will now fire up various micro-services."
 	@echo
 	@echo " - Run \`make log\` in another terminal to view logs"
 	@echo " - Run \`make monitor\` in another terminal to restart services"
@@ -71,7 +74,6 @@ run_testing: paths dependencies
 	$(ENV_SUMMARY) && \
 	$(SUPERVISORD)
 
-
 monitor:
 	@echo "Entering supervisor control panel."
 	@echo " - Type \`status\` too see microservice status"
@@ -81,31 +83,13 @@ monitor:
 attach:
 	$(SUPERVISORCTL) fg app
 
-run_production:
-	export FLAGS="--config config.yaml" && \
-	$(ENV_SUMMARY) && \
-	$(SUPERVISORD)
-
-run_testing: paths dependencies
-	export FLAGS="--config _test_config.yaml" && \
-	$(ENV_SUMMARY) && \
-	$(SUPERVISORD)
-
-debug:
-	@echo "Starting web service in debug mode"
-	@echo "Press Ctrl-D to stop"
-	@echo
-	@export FLAGS="--debug" && $(SUPERVISORD) &
-	@sleep 1 && $(SUPERVISORCTL) -i status
-	@$(SUPERVISORCTL) shutdown
-
 clean:
 	rm $(bundle)
 
-test_headless: paths dependencies
+test_headless: paths dependencies fill_conf_values
 	PYTHONPATH='.' xvfb-run ./tools/test_frontend.py
 
-test: paths dependencies
+test: paths dependencies fill_conf_values
 	PYTHONPATH='.' ./tools/test_frontend.py
 
 stop:

baselayer

@@ -79,7 +79,7 @@ def post(self):
         DBSession().add(d)
         DBSession().commit()
 
-        return self.success(d, 'cesium/FETCH_DATASETS')
+        return self.success(d.display_info(), 'cesium/FETCH_DATASETS')
 
     @auth_or_token
     def get(self, dataset_id=None):

cesium_app/handlers/dataset.py

@@ -1,6 +1,6 @@
 import textwrap
 
-from baselayer.app.config import load_env
+from baselayer.app.env import load_env
 from baselayer.app.model_util import status, create_tables, drop_tables
 from cesium_app import models
 
@@ -58,6 +58,16 @@ def insert_test_data():
         models.DBSession().commit()
 
 
+def create_token_user(bot_name, project_ids):
+    u = models.User(username=bot_name)
+    p = models.Project.query.filter(models.Project.id.in_(project_ids)).all()
+    u.projects.extend(p)
+    t = models.Token(user=u)
+    models.DBSession().add_all([u, t])
+    models.DBSession().commit()
+    return t.id
+
+
 if __name__ == "__main__":
     env, cfg = load_env()
 

cesium_app/model_util.py

@@ -9,6 +9,8 @@
 import numpy.testing as npt
 import pandas as pd
 import json
+import subprocess
+from cesium_app.model_util import create_token_user
 
 
 def _add_prediction(proj_id, driver):
@@ -221,27 +223,30 @@ def test_download_prediction_csv_regr(driver, project, dataset, featureset, mode
 
 
 def test_predict_specific_ts_name(driver, project, dataset, featureset, model):
+    auth_token = create_token_user(uuid.uuid4(), [project.id])
+
     ts_data = [[1, 2, 3, 4], [32.2, 53.3, 32.3, 32.52], [0.2, 0.3, 0.6, 0.3]]
     impute_kwargs = {'strategy': 'constant', 'value': None}
     data = {'datasetID': dataset.id,
             'ts_names': ['217801'],
-            'modelID': model.id}
+            'modelID': model.id,
+            'token': auth_token}
     response = driver.request(
         'POST', '{}/predictions'.format(driver.server_url),
-        data=json.dumps(data)).json()
+        json=data).json()
+    print(response)
     assert response['status'] == 'success'
 
-    driver.request('GET', f'{driver.server_url}')  # ensure login
-
     for i in range(10):
         pred_info = driver.request('GET', '{}/predictions/{}'.format(
-            driver.server_url, response['data']['id'])).json()
+            driver.server_url, response['data']['id']),
+                                   json={'token': auth_token}).json()
         if pred_info['status'] == 'success' and pred_info['data']['finished']:
-            assert isinstance(pred_info['data']['results']['217801']
-                              ['features']['total_time'],
-                              float)
-            assert 'Mira' in pred_info['data']['results']['217801']['prediction']
             break
         time.sleep(1)
     else:
         raise Exception('test_predict_specific_ts_name timed out')
+    assert isinstance(pred_info['data']['results']['217801']
+                      ['features']['total_time'],
+                      float)
+    assert 'Mira' in pred_info['data']['results']['217801']['prediction']

cesium_app/tests/frontend/test_predict.py

@@ -44,3 +44,13 @@ server:
         debug_login: True
         google_oauth2_key:
         google_oauth2_secret:
+
+ports:
+    websocket: 64000
+    fake_oauth: 63000
+    app: 5000
+    app_http_proxy: 5001
+    app_internal: 65000
+    dask: 63500
+    websocket_path_in: 'ipc:///tmp/message_flow_in'
+    websocket_path_out: 'ipc:///tmp/message_flow_out'