Mervin Praison

pip install torch transformers torchaudio gradio

import gradio as gr
from transformers import pipeline
import numpy as np

transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-base.en")

def transcribe(audio):
    # Check if audio input exists
    if audio is None:
        return "Please record some audio"
        
    sr, y = audio
    
    # Convert to mono if stereo
    if y.ndim > 1:
        y = y.mean(axis=1)
        
    y = y.astype(np.float32)
    y /= np.max(np.abs(y))

    return transcriber({"sampling_rate": sr, "raw": y})["text"]  

demo = gr.Interface(
    transcribe,
    gr.Audio(sources="microphone", type="numpy"),  # Specify type as numpy
    "text",
)

demo.launch()

Realtime Streaming (not working)

import gradio as gr
from transformers import pipeline
import numpy as np

transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-base.en")

def transcribe(stream, new_chunk):
    if new_chunk is None:
        return None, ""
        
    sr, y = new_chunk
    
    # Convert to mono if stereo
    if y.ndim > 1:
        y = y.mean(axis=1)
        
    y = y.astype(np.float32)
    y /= np.max(np.abs(y)) + 1e-7  # Safe normalization

    # Initialize or update stream
    if stream is not None:
        stream = np.concatenate([stream, y])
    else:
        stream = y
        
    # Keep only last 5 seconds to prevent memory issues
    max_length = sr * 5
    if len(stream) > max_length:
        stream = stream[-max_length:]
        
    try:
        text = transcriber({"sampling_rate": sr, "raw": stream})["text"]
        return stream, text
    except Exception as e:
        print(f"Transcription error: {e}")
        return stream, ""

demo = gr.Interface(
    fn=transcribe,
    inputs=[
        "state",
        gr.Audio(sources=["microphone"], streaming=True, type="numpy")
    ],
    outputs=[
        "state",
        "text"
    ],
    live=True
)

if __name__ == "__main__":
    demo.queue().launch()

Download model script

download.sh

#!/bin/bash

# Base URL for downloading model files
BASE_URL="https://huggingface.co/microsoft/OmniParser/resolve/main"

# Define folder structure and create folders
mkdir -p weights/icon_detect
mkdir -p weights/icon_caption_florence
mkdir -p weights/icon_caption_blip2

# Declare an associative array of required files with paths
declare -A model_files=(
  ["weights/icon_detect/model.safetensors"]="$BASE_URL/icon_detect/model.safetensors"
  ["weights/icon_detect/model.yaml"]="$BASE_URL/icon_detect/model.yaml"
  ["weights/icon_caption_florence/model.safetensors"]="$BASE_URL/icon_caption_florence/model.safetensors"
  ["weights/icon_caption_florence/config.json"]="$BASE_URL/icon_caption_florence/config.json"
  ["weights/icon_caption_blip2/model.safetensors"]="$BASE_URL/icon_caption_blip2/model.safetensors"
  ["weights/icon_caption_blip2/config.json"]="$BASE_URL/icon_caption_blip2/config.json"
)

# Download each file into its specified directory
for file_path in "${!model_files[@]}"; do
  wget -O "$file_path" "${model_files[$file_path]}"
done

echo "All required model and configuration files downloaded and organised."

# Run the conversion script if necessary files are present
if [ -f "weights/icon_detect/model.safetensors" ] && [ -f "weights/icon_detect/model.yaml" ]; then
  python weights/convert_safetensor_to_pt.py
  echo "Conversion to best.pt completed."
else
  echo "Error: Required files for conversion not found."
fi

bash download.sh

Python code

app.py

# 1. Import libraries

from utils import get_som_labeled_img, check_ocr_box, get_caption_model_processor, get_yolo_model
import torch
from ultralytics import YOLO
from PIL import Image
import base64
import matplotlib.pyplot as plt
import io
import json
from rich import print

# 2. Configuration

device = 'cuda'
som_model = get_yolo_model(model_path='weights/icon_detect/best.pt')
som_model.to(device)
print('model to {}'.format(device))
caption_model_processor = get_caption_model_processor(model_name="florence2", model_name_or_path="weights/icon_caption_florence", device=device)

image_path = 'imgs/windows_multitab.png'
image = Image.open(image_path)
image_rgb = image.convert('RGB')

draw_bbox_config = {
    'text_scale': 0.8,
    'text_thickness': 2,
    'text_padding': 3,
    'thickness': 3,
}
BOX_TRESHOLD = 0.03

# 3. Get labeled image and results
ocr_bbox_rslt, is_goal_filtered = check_ocr_box(
    image_path, 
    display_img=False, 
    output_bb_format='xyxy', 
    goal_filtering=None, 
    easyocr_args={'paragraph': False, 'text_threshold': 0.9}
)
text, ocr_bbox = ocr_bbox_rslt

dino_labeled_img, label_coordinates, parsed_content_list = get_som_labeled_img(
    image_path, 
    som_model, 
    BOX_TRESHOLD=BOX_TRESHOLD, 
    output_coord_in_ratio=False, 
    ocr_bbox=ocr_bbox, 
    draw_bbox_config=draw_bbox_config, 
    caption_model_processor=caption_model_processor, 
    ocr_text=text, 
    use_local_semantics=True, 
    iou_threshold=0.1
)

decoded_image = Image.open(io.BytesIO(base64.b64decode(dino_labeled_img)))
decoded_image.save("labeled_image_output.png")
print(parsed_content_list)

python app.py

# 1. System Updates and Dependencies
sudo apt-get update
sudo apt-get install -y build-essential
sudo apt-get install -y libstdc++-12-dev
sudo apt-get install -y clang llvm
sudo apt-get install -y libc++-dev libc++abi-dev
sudo apt-get install -y llvm-14 llvm-14-dev
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"

# 2. Conda Environment Setup
conda create -n bitnet-cpp python=3.9 -y
conda activate bitnet-cpp

# 3. Clone Repository
git clone --recursive https://github.com/microsoft/BitNet.git
cd BitNet

# 4. Python Dependencies
pip install -r requirements.txt
pip install cmake

# 5. Download Model
huggingface-cli download HF1BitLLM/Llama3-8B-1.58-100B-tokens --local-dir models/Llama3-8B-1.58-100B-tokens

# 6. Run Setup
python setup_env.py -md models/Llama3-8B-1.58-100B-tokens -q i2_s

# 7. Run Inference
python run_inference.py -m models/Llama3-8B-1.58-100B-tokens/ggml-model-i2_s.gguf -p "Daniel went back to the the the garden. Mary travelled to the kitchen. Sandra journeyed to the kitchen. Sandra went to the hallway. John went to the bedroom. Mary went back to the garden. Where is Mary?\nAnswer:" -n 6 -temp 0