Merge pull request #145 from nnnyt/update_doc

[DOC] reorganize downstream tasks

Author: KenelmQLH

examples/downstream/difficulty_prediction.ipynb renamed to examples/downstream/difficulty_prediction/difficulty_prediction.ipynb

@@ -1,24 +1,14 @@
 import json
 import pandas as pd
 
+
 def load_json(open_path):
     print("[load_json] start : {}".format(open_path))
     with open(open_path, "r", encoding='utf-8') as f:
         load_q = json.load(f)
     print("[load_json] num = {}, open_path = {}".format(len(load_q), open_path))
     return load_q
 
-def pre_disc(csv_path):
-    items = pd.read_csv(csv_path)
-    stem = items["stem"].tolist()
-    disc = items["disc"].tolist()
-    data = []
-    for i in range(len(stem)):
-        dic = {}
-        dic["content"] = stem[i]
-        dic["labels"] = disc[i]
-        data.append(dic)
-    return data
 
 def get_train(train):
     train_data = []
@@ -29,6 +19,7 @@ def get_train(train):
         train_data.append(dic)
     return train_data
 
+
 def get_val(val):
     test_data, test_gap = [], []
     start, end = 0, 0
@@ -38,12 +29,8 @@ def get_val(val):
             dic = {}
             dic['content'] = item["stem"]
             dic['labels'] = item['diff']
-            #dic["labels"] = dic.pop("difficulty")
+            # dic["labels"] = dic.pop("difficulty")
             test_data.append(dic)
-        test_gap.append([start, end])    
+        test_gap.append([start, end])
         start = end
     return test_data, test_gap
-    
-
-
-    

examples/downstream/utils.py renamed to examples/downstream/difficulty_prediction/utils.py

@@ -0,0 +1,15 @@
+import json
+import pandas as pd
+
+
+def pre_disc(csv_path):
+    items = pd.read_csv(csv_path)
+    stem = items["stem"].tolist()
+    disc = items["disc"].tolist()
+    data = []
+    for i in range(len(stem)):
+        dic = {}
+        dic["content"] = stem[i]
+        dic["labels"] = disc[i]
+        data.append(dic)
+    return data