checkpoint

Author: brandonjbjelland

AUTHORS.md

@@ -7,6 +7,7 @@ If you are submitting a change, please add your name or the name of the organiza
 ## Humans
 
 * [Brandon Bjelland](https://github.com/brandonjbjelland) \<brandon at atscale.run\>
+* [Orrin Fiandaca](https://github.com/orrin) \<ofiandaca at gmail.com\>
 
 ## Organizations
 

CHANGELOG.md

@@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file. See [standa
 - Snippets updated to match latest documentation. Created snippets 5385 from 111 providers.
 - providers now sourced from the public registry
 - upgraded all libaries
+- refactored provider fetch workflow to remove the need for the hard-coded number of total snippets
 
 ## [0.4.0](https://github.com/run-at-scale/vscode-terraform-doc-snippets/compare/v0.3.0...v0.4.0) (2019-03-27)
 

README.md

@@ -42,7 +42,6 @@ npm run build:snippets
 ## Known Issues
 
 - Issue #1 is probably that this is my first node project and I'm not to be trusted with the language. Help and review wanted! Tests needed.
-- The async nature of node means we need to provide a hard stop at the number of snippets collected before cleanup can happen. The stop-gap implementation has this count specified by hand in `config.json`. If a run of `build:snippets` produces more than the value of `total_snippets`, adjust the number up to match.
 
 ## Contributing
 

src/config.json

@@ -1,5 +1,4 @@
 {
-  "total_snippets": 5385,
   "override_snippets": [
     {
       "tf-conditional": {

src/downloadGitRepo.ts

@@ -22,7 +22,7 @@ class SnippetClass {
   }
 }
 
-export function downloadGitRepo(
+export async function downloadGitRepo(
   workspace,
   providerRepo,
   providerName,
@@ -31,7 +31,7 @@ export function downloadGitRepo(
 ) {
   console.log(`Downloading git repo for ${providerName} at ${providerRepo}`);
   const providerDir = `${workspace}/${providerName}`;
-  git.Clone.clone(providerRepo, providerDir).then(() => {
+  await git.Clone.clone(providerRepo, providerDir).then(() => {
     async.each(Object.keys(tfTypesMap), (tfType) => {
       const providerTypeDir = `${providerDir}/website/docs/${tfType}`;
       const resourceType = tfTypesMap[tfType];

src/getAvailableProviders.ts

@@ -1,26 +1,40 @@
 import request = require('request');
-import async = require("async");
-import { iterateOnDocFiles } from "./iterateOnDocFiles";
+
 const urlBase = 'https://registry.terraform.io'
 
-// handle errors and do pagination
-export function getAvailableProviders(tmpDir, snips, downloadGitRepo) {
+export async function getAvailableProviders() {
   const urlSuffix = '/v1/providers?offset=0'
-  fetchProviderBatch(urlBase + urlSuffix, snips, tmpDir, downloadGitRepo)
-}
+  let result;
+  let url;
+  const list = [];
 
-function fetchProviderBatch(url, snips, tmpDir, downloadGitRepo) {
-  console.log(url)
-  request(url, { json: true }, (err, _, body) => {
-    if (err) {
-        return console.log(err);
+  do {
+    // API paging
+    if (result.hasOwnProperty('meta')) {
+      //repeat API call for page 2+
+      url = urlBase + result.meta.next_url;
+      result = await request(url, { json: true }, (err, _, body) => {
+        if (err) {
+            return console.log(err);
+        }
+        return body
+      });
     }
-    let nextUrl = urlBase + body.meta.next_url
-    if (nextUrl != url) {
-      fetchProviderBatch(nextUrl, snips, tmpDir, downloadGitRepo)
+    else {
+      //first API call - get page 1
+      url = urlBase + urlSuffix;
+      result = await request(url, { json: true }, (err, _, body) => {
+        if (err) {
+            return console.log(err);
+        }
+        return body
+      });
     }
-    async.forEach(body.providers, (element) => {
-      downloadGitRepo(tmpDir, element.source, element.name, snips, iterateOnDocFiles)
+    console.log(result)
+    result.providers.forEach((element) => {
+      list.push(element.source);
     });
-  });
+  } while (urlBase + result.meta.next_url != url);
+
+  return list;
 }

src/main.ts

@@ -1,36 +1,49 @@
 import { writeFileSync } from "fs";
 import { cleanup } from "./cleanup";
 import { downloadGitRepo } from "./downloadGitRepo";
+import { iterateOnDocFiles } from "./iterateOnDocFiles";
 import { getAvailableProviders } from "./getAvailableProviders";
 import { setOverrides } from "./setOverrides";
 import setup = require("./setup");
 import { sortObject } from "./sortObject";
+import { exit } from "process";
+
 const configuration = require("./config.json");
 
 async function main() {
-  const cap = configuration.total_snippets;
   const tmpDir = setup.setupWorkspace();
   const snips = {};
-  getAvailableProviders(tmpDir, snips, downloadGitRepo);
-  // TODO: find a better way to count up front the number of processed markdown files OR providers
-  await sleep(600);
-  while (Object.keys(snips).length < cap) {
-    console.log(`${Object.keys(snips).length} of ${cap}`);
-    await sleep(20);
-  }
-  console.log(Object.keys(snips).length);
-  const overrideSnipsIncluded = setOverrides(snips);
-  const sortedSnips = sortObject(overrideSnipsIncluded);
-  writeFileSync(
-    "snippets/terraform.json",
-    JSON.stringify(sortedSnips, null, 2),
-    "utf-8",
-  );
-  cleanup(tmpDir);
-}
+  const providersList = await getAvailableProviders();
+  console.log(providersList);
+  exit
+
+  // for(let i = 0; i < providersList.length; i++) {
+  //   const name = providersList[i];
+  //   let subtotal = Object.keys(snips).length;
+
+  //   await downloadGitRepo(tmpDir, name, snips, iterateOnDocFiles);
+
+  //   subtotal = Object.keys(snips).length - subtotal;
+  //   console.log(`provider ${i + 1} of ${providersList.length}: ${subtotal} snips\n`)
+
+
 
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms * 100));
+  // getAvailableProviders(tmpDir, snips, downloadGitRepo);
+  // // TODO: find a better way to count up front the number of processed markdown files OR providers
+  // await sleep(600);
+  // while (Object.keys(snips).length < cap) {
+  //   console.log(`${Object.keys(snips).length} of ${cap}`);
+  //   await sleep(20);
+  // }
+  // console.log(Object.keys(snips).length);
+  // const overrideSnipsIncluded = setOverrides(snips);
+  // const sortedSnips = sortObject(overrideSnipsIncluded);
+  // writeFileSync(
+  //   "snippets/terraform.json",
+  //   JSON.stringify(sortedSnips, null, 2),
+  //   "utf-8",
+  // );
+  // cleanup(tmpDir);
 }
 
 main();