dev

app.py

@@ -1,20 +1,39 @@
-#!/usr/bin/env python#
+#!/usr/bin/env python
+
+import os
+import requests
+from threading import Thread
+from typing import Iterator
 
-from llama_cpp import Llama
-from time import time
 import gradio as gr
 import psutil
-import os
+import spaces
+import torch
+from time import time
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from llama_cpp import Llama
 
 # load like this - use tne variable everywhere 
-model_hf_path=os.getenv("MODEL_HF_PATH")
+model_uri_hf=os.getenv("MODEL_URI_HF")
+# DEBUG!
+model_uri_hf="https://huggingface.co/TheBloke/neural-chat-7B-v3-2-GGUF/blob/main/neural-chat-7b-v3-2.Q2_K.gguf"
 # show warning, when empty and briefs description of how to set it 
 # also add link to "how to search" with link to bloke by default + example search link + example full value (mistral base?)
 # info about ram requirements
 
 # Initing things                
-print(f"debug: init model: {model_hf_path}")
-#llm = Llama(model_path="./model.bin")                             # LLaMa model
+print(f"debug: init model: {model_uri_hf}")
+
+# Check if the model file already exists
+if not os.path.isfile('model.bin'):
+    # Download the model
+    response = requests.get(model_uri_hf)
+
+    # Save the model to a local file
+    with open('model.bin', 'wb') as file:
+        file.write(response.content)
+
+llm = Llama(model_path="./model.bin")                             # LLaMa model
 print("! INITING DONE !")
 
 # Preparing things to work
@@ -31,27 +50,18 @@ print(f"DEBUG: Memory free: {psutil.virtual_memory().free / (1024.0 ** 3)} GiB")
 print(f"DEBUG: Memory available: {psutil.virtual_memory().available / (1024.0 ** 3)} GiB")
 print(f"DEBUG: Memory: {psutil.virtual_memory().total / (1024.0 ** 3)} GiB")
 
-
-from threading import Thread
-from typing import Iterator
-
-import spaces
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-
-DESCRIPTION =  f"# Test model: {model_hf_path}"
+DESCRIPTION =  f"# Test model: {model_uri_hf}"
 
 if torch.cuda.is_available():
     DESCRIPTION += "\n<p>This space is using CPU only. Use a different one if you want to go fast and use GPU. </p>"
 
+#todo - probably lower. like 200 in and maybe 500 out? Should be ok for quick test
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
 MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 
 
-#download model here
-# check localstorage, if no there, load, else use existing.
-# check gradio - how does it dl? is there a function we can use?
+
 
 if torch.cuda.is_available():
     model_id = "mistralai/Mistral-7B-Instruct-v0.1"
@@ -79,6 +89,7 @@ def generate(
         gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
     input_ids = input_ids.to(model.device)
 
+    streamer= Llama()
     streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
     generate_kwargs = dict(
         {"input_ids": input_ids},