update tutorial and api references

Author: karin0018

EduNLP/I2V/i2v.py

@@ -97,19 +97,43 @@ def __call__(self, items, *args, **kwargs):
         return self.infer_vector(items, *args, **kwargs)
 
     def tokenize(self, items, *args, key=lambda x: x, **kwargs) -> list:
-        # """tokenize item"""
+        """
+        tokenize item
+        
+        Parameter
+        ----------
+        items: a list of questions
+        
+        Return
+        ----------
+        tokens: list
+        """
         return self.tokenizer(items, *args, key=key, **kwargs)
 
     def infer_vector(self, items, key=lambda x: x, **kwargs) -> tuple:
+        """
+        get question embedding
+        
+        NotImplemented
+        """
         raise NotImplementedError
 
     def infer_item_vector(self, tokens, *args, **kwargs) -> ...:
+        """NotImplemented"""
         return self.infer_vector(tokens, *args, **kwargs)[0]
 
     def infer_token_vector(self, tokens, *args, **kwargs) -> ...:
+        """NotImplemented"""
         return self.infer_vector(tokens, *args, **kwargs)[1]
 
     def save(self, config_path):
+        """
+        save model weights in config_path
+        
+        Parameter:
+        ----------
+        config_path: str
+        """
         with open(config_path, "w", encoding="utf-8") as wf:
             json.dump(self.params, wf, ensure_ascii=False, indent=2)
 
@@ -126,6 +150,7 @@ def load(cls, config_path, *args, **kwargs):
 
     @classmethod
     def from_pretrained(cls, name, model_dir=MODEL_DIR, *args, **kwargs):
+        """NotImplemented"""
         raise NotImplementedError
 
     @property

EduNLP/Vector/disenqnet/disenqnet.py

@@ -44,6 +44,16 @@ def infer_vector(self, items: dict, vector_type=None, **kwargs) -> torch.Tensor:
 
     def infer_tokens(self, items: dict, **kwargs) -> torch.Tensor:
         embeded, _, _ = self(items)
+        """
+        get tokens embedding with DisenQModel
+        
+        Parameters
+        ----------
+        items: dict, {'content_idx': tensor(),'content_len': tensor()}, the tokens about question items after tokenizer processing
+
+        Returns:
+            torch.Tensor: token embedding
+        """
         return embeded
 
     @property

EduNLP/Vector/elmo_vec.py

@@ -21,6 +21,16 @@ def __call__(self, items: dict):
         return outputs
 
     def infer_vector(self, items: dict, **kwargs) -> torch.Tensor:
+        """
+        get sentence vector embedding with ElmoModel
+        
+        Parameters
+        ----------
+        items: dict, {'seq_idx': tensor(),'seq_len':tensor()}, the tokens about question after tokenizer processing
+
+        Returns:
+            torch.Tensor: sentence embedding
+        """
         outputs = self(items)
         item_embeds = torch.cat(
             (outputs.forward_output[torch.arange(len(items["seq_len"])), torch.tensor(items["seq_len"]) - 1],
@@ -29,6 +39,16 @@ def infer_vector(self, items: dict, **kwargs) -> torch.Tensor:
         return item_embeds
 
     def infer_tokens(self, items, **kwargs) -> torch.Tensor:
+        """
+        get tokens embedding with ElmoModel
+        
+        Parameters
+        ----------
+        items: dict, {'seq_idx': tensor()}, the tokens about question after tokenizer processing
+
+        Returns:
+            torch.Tensor: token embedding
+        """
         outputs = self(items)
         forward_hiddens = outputs.forward_output
         backward_hiddens = outputs.backward_output

EduNLP/Vector/gensim_vec.py

@@ -66,11 +66,36 @@ def __getitem__(self, item):
         return self.wv[item] if index not in self.constants.values() else np.zeros((self.vector_size,))
 
     def infer_vector(self, items, agg="mean", **kwargs) -> list:
+        """
+        get sentence embedding with word2vec model
+        
+        Parameters
+        ----------
+        item: list, the tokens after tokenizer processing
+        
+        Return
+        ------
+        vector: list
+            [array(), ..., array()]
+        """
         token_vectors = self.infer_tokens(items, **kwargs)
         # return [eval("np.%s" % agg)(item, axis=0) if item else np.array([]) for item in token_vectors]
         return [eval("np.%s" % agg)(item, axis=0) if item else np.zeros(self.vector_size,) for item in token_vectors]
 
     def infer_tokens(self, items, **kwargs) -> list:
+        """
+        get token embedding with word2vec model
+        
+        Parameters
+        ----------
+        item: list
+            the tokens after tokenizer processing
+        
+        Return
+        ------
+        vector: list  
+            [[array(), ..., array()], [...], [...]]
+        """
         return [list(self(*item)) for item in items]
 
 
@@ -95,6 +120,19 @@ def __init__(self, filepath):
         self.dictionary = corpora.Dictionary.load(filepath)
 
     def infer_vector(self, item, return_vec=False):
+        """
+        get Bow vector
+        
+        Parameters
+        ----------
+        item: list
+            the tokens after tokenizer processing
+        
+        Return
+        ------
+        vector: list
+            [array(), ..., array()]
+        """        
         item = self.dictionary.doc2bow(item)
         if not return_vec:
             return item  # return dic as default
@@ -121,6 +159,19 @@ def __init__(self, filepath):
         self.dictionary = corpora.Dictionary.load(dictionary_path)
 
     def infer_vector(self, item, return_vec=False):
+        """
+        get Tf-idf vector
+        
+        Parameters
+        ----------
+        item: list
+            the tokens after tokenizer processing
+        
+        Return
+        ------
+        vector: list 
+            [array(), ..., array()]
+        """
         dic_item = self.dictionary.doc2bow(item)
         tfidf_item = self.tfidf_model[dic_item]
         # return dic as default
@@ -181,7 +232,25 @@ def vector_size(self):
             return self.d2v.vector_size
 
     def infer_vector(self, items, *args, **kwargs) -> list:
+        """
+        get vector with D2V model
+        
+        Parameters
+        ----------
+        item: list  
+            the tokens after tokenizer processing
+        
+        Return
+        ------
+        vector: list
+            [array(), ..., array()]
+        """
         return [self(item) for item in items]
 
     def infer_tokens(self, item, *args, **kwargs) -> ...:
+        """
+        get token embeddings with D2V
+        
+        NotImplemented
+        """
         raise NotImplementedError

EduNLP/Vector/t2v.py

@@ -49,8 +49,10 @@ class T2V(object):
     ... 如图$\\FigureID{088f15ea-8b7c-11eb-897e-b46bfc50aa29}$,若$x,y$满足约束条件$\\SIFSep$，则$z=x+7 y$的最大值为$\\SIFBlank$'}]
     >>> model_dir = "examples/test_model/d2v"
     >>> url, model_name, *args = get_pretrained_model_info('d2v_test_256')
-    >>> (); path = get_data(url, model_dir); () # doctest: +ELLIPSIS
-    (...)
+    >>> path = get_data(url, model_dir); # doctest: +ELLIPSIS
+    downloader, INFO http://base.ustc.edu.cn/data/model_zoo/modelhub/doc2vec_pub/1/d2v_test_256.zip is saved as examples\test_model\d2v\d2v_test_256.zip
+    Downloading examples\test_model\d2v\d2v_test_256.zip 100.00%: 4.73MB | 4.73MB
+    downloader, INFO examples\test_model\d2v\d2v_test_256.zip is unzip to examples\test_model\d2v\d2v_test_256
     >>> path = path_append(path, os.path.basename(path) + '.bin', to_str=True)
     >>> t2v = T2V('d2v',filepath=path)
     >>> print(t2v(item))
@@ -69,9 +71,28 @@ def __call__(self, items, *args, **kwargs):
         return self.i2v.infer_vector(items, *args, **kwargs)
 
     def infer_vector(self, items, *args, **kwargs):
+        """
+        get question embedding with T2V
+        
+        Parameters
+        ----------
+        items:list
+            a list of question 
+        
+        Returns
+        -------
+        vector:list
+            numpy.ndarray([dtype=float32)]
+        
+        """
         return self.i2v.infer_vector(items, *args, **kwargs)
 
     def infer_tokens(self, items, *args, **kwargs):
+        """
+        get token embeddings with T2V
+        
+        NotImplemented
+        """
         return self.i2v.infer_tokens(items, *args, **kwargs)
 
     @property
@@ -80,6 +101,27 @@ def vector_size(self) -> int:
 
 
 def get_pretrained_model_info(name):
+    """
+    get the pretrained model information with the given name
+    
+    Parameters
+    ----------
+    name:str
+        select the pretrained model
+        e.g.:
+        d2v_math_300
+        w2v_math_300
+        elmo_math_2048
+        bert_math_768
+        bert_taledu_768
+        disenq_math_256
+        quesnet_math_512
+    
+    Returns
+    --------
+    list: [model url (where to download),  model name]
+    
+    """
     url = MODELHUB_URL + 'getPretrainedModel'
     param = {'name': name}
     r = requests.get(url, params=param)
@@ -89,6 +131,14 @@ def get_pretrained_model_info(name):
 
 
 def get_all_pretrained_models():
+    """
+    get all pretrained models' name
+
+    Returns
+    -------
+    the pretrained models' name:list
+        e.g.['bert_bio_ptc', 'bert_geo_ptc', 'bert_math_768', ... ]
+    """
     url = MODELHUB_URL + 'getPretrainedModelList'
     r = requests.get(url)
     assert r.status_code == 200, r.status_code

docs/README.md

@@ -3,26 +3,31 @@ EduNLP document and tutorial folder
 
 Requirements
 ------------
+
 See the requirements `docs_deps` in `setup.py`:
+
 ```sh
 pip install -e .[doc]
 ```
 
-
 Build documents
 ---------------
+
 First, clean up existing files:
+
 ```
 make clean
 ```
 
 Then build:
+
 ```
 make html
 ```
 
 Render locally
 --------------
+
 ```
 cd build/html
 python3 -m http.server 8000

docs/source/api/vector.rst

@@ -10,13 +10,13 @@ EduNLP.Vector.t2v
 
 
 EduNLP.Vector.disenqnet
---------------------
+-------------------------
 
 .. automodule:: EduNLP.Vector.disenqnet.disenqnet
    :members:
 
 EduNLP.Vector.quesnet
---------------------
+-------------------------
 
 .. automodule:: EduNLP.Vector.quesnet.quesnet
    :members:

docs/source/conf.py

@@ -114,3 +114,5 @@ def copy_tree(src, tar):
     'undoc-members': True,
 }
 autodoc_member_order = 'bysource'
+
+nbsphinx_allow_errors = True

docs/source/tutorial/zh/pipeline.rst

@@ -2,9 +2,7 @@
 流水线
 =======
 
-.. nbgallery::
-    :caption: This is a thumbnail gallery:
-    :name: pipleine_gallery
-    :glob:
+.. nbinfo::
+    notebook: 
 
-    流水线  <../../build/blitz/pipeline/pipeline.ipynb>
+    `流水线 <../../build/blitz/pipeline/pipeline.ipynb>`_