update

util.py

@@ -72,6 +72,7 @@ def tokenize_pair(text, tokenizer_type_1, tokenizer_type_2):
     return pos_tokens_1, table_df_1, pos_tokens_2, table_df_2
 
 
+@lru_cache
 def basic_count(tokenizer_type):
     tokenizer = load_tokener(tokenizer_type)
     stats = iter_vocab(tokenizer, tokenizer_type)
@@ -113,7 +114,6 @@ def on_load(url_params, request: gr.Request):
     """
     onLoad
     """
-
     text = None
     tokenizer_type_1 = None
     tokenizer_type_2 = None
@@ -122,11 +122,7 @@ def on_load(url_params, request: gr.Request):
     except:
         url_params = {}
     if request:
-        try:
-            logger.info(str(request.headers))
-            logger.info(str(request.query_params))
-        except:
-            pass
+        logger.info(str(request.headers))
         client_ip = request.client.host
         # local_ip = socket.gethostbyname(socket.gethostbyname(""))
         # headers = request.kwargs['headers']
@@ -139,8 +135,6 @@ def on_load(url_params, request: gr.Request):
         tokenizer_type_1 = url_params.get("tokenizer1", default_tokenizer_type_1)
         tokenizer_type_2 = url_params.get("tokenizer2", default_tokenizer_type_2)
         text = url_params.get("text", default_user_input)
-
-
         logger.info(f"client_ip: {client_ip}; params: {url_params}")
     return text, tokenizer_type_1, tokenizer_type_2
 

vocab/__init__.py

@@ -75,7 +75,7 @@ all_tokenizers = [
     # ##### glm系列
     "glm_chinese",
     "chatglm_6b",
-    "chatglm2-6b",
+    "chatglm2_6b",
     #
     # #### llama alpaca系列
     "llama",  # '中文单字': 700, '中文多字': 0