Update app.py
Browse files
app.py
CHANGED
|
@@ -6,7 +6,7 @@ from diarizationlm import utils
|
|
| 6 |
title = "DiarizationLM GGUF inference on CPU"
|
| 7 |
|
| 8 |
description = """
|
| 9 |
-
A demo of the DiarizationLM model finetuned from Llama 3. In this demo, we run a 4-bit quantized GGUF model on CPU.
|
| 10 |
|
| 11 |
To learn more about DiarizationLM, check our paper: https://arxiv.org/abs/2401.03506
|
| 12 |
"""
|
|
@@ -16,19 +16,19 @@ model_name = "q4_k_m.gguf"
|
|
| 16 |
prompt_suffix = " --> "
|
| 17 |
completion_suffix = " [eod]"
|
| 18 |
|
| 19 |
-
hf_hub_download(repo_id="google/DiarizationLM-8b-Fisher-v2", filename=model_name, local_dir=model_path
|
| 20 |
|
| 21 |
print("Start the model init process")
|
| 22 |
model = GPT4All(model_name=model_name,
|
| 23 |
model_path=model_path,
|
| 24 |
-
allow_download = False,
|
| 25 |
device="cpu")
|
| 26 |
print("Finish the model init process")
|
| 27 |
|
| 28 |
def generater(message, history):
|
| 29 |
prompt = message + prompt_suffix
|
| 30 |
max_new_tokens = round(len(prompt) / 3.0 * 1.2)
|
| 31 |
-
outputs = []
|
| 32 |
for token in model.generate(prompt=prompt,
|
| 33 |
temp=0.1,
|
| 34 |
top_k=50,
|
|
@@ -42,7 +42,7 @@ def generater(message, history):
|
|
| 42 |
break
|
| 43 |
transferred_completion = utils.transfer_llm_completion(completion, message)
|
| 44 |
yield transferred_completion
|
| 45 |
-
|
| 46 |
print("Create chatbot")
|
| 47 |
chatbot = gr.Chatbot()
|
| 48 |
print("Created chatbot")
|
|
|
|
| 6 |
title = "DiarizationLM GGUF inference on CPU"
|
| 7 |
|
| 8 |
description = """
|
| 9 |
+
A demo of the DiarizationLM model finetuned from Llama 3. In this demo, we run a 4-bit quantized GGUF model on CPU.
|
| 10 |
|
| 11 |
To learn more about DiarizationLM, check our paper: https://arxiv.org/abs/2401.03506
|
| 12 |
"""
|
|
|
|
| 16 |
prompt_suffix = " --> "
|
| 17 |
completion_suffix = " [eod]"
|
| 18 |
|
| 19 |
+
hf_hub_download(repo_id="google/DiarizationLM-8b-Fisher-v2", filename=model_name, local_dir=model_path)
|
| 20 |
|
| 21 |
print("Start the model init process")
|
| 22 |
model = GPT4All(model_name=model_name,
|
| 23 |
model_path=model_path,
|
| 24 |
+
allow_download = False,
|
| 25 |
device="cpu")
|
| 26 |
print("Finish the model init process")
|
| 27 |
|
| 28 |
def generater(message, history):
|
| 29 |
prompt = message + prompt_suffix
|
| 30 |
max_new_tokens = round(len(prompt) / 3.0 * 1.2)
|
| 31 |
+
outputs = []
|
| 32 |
for token in model.generate(prompt=prompt,
|
| 33 |
temp=0.1,
|
| 34 |
top_k=50,
|
|
|
|
| 42 |
break
|
| 43 |
transferred_completion = utils.transfer_llm_completion(completion, message)
|
| 44 |
yield transferred_completion
|
| 45 |
+
|
| 46 |
print("Create chatbot")
|
| 47 |
chatbot = gr.Chatbot()
|
| 48 |
print("Created chatbot")
|