Chinese CLIP原理及实践(更新中)

November丶Chopin

已于 2023-04-24 11:42:53 修改

阅读量3.4k

点赞数

分类专栏：专栏06-Deep Learning 文章标签：深度学习文搜图多模态

于 2023-03-17 11:05:26 首次发布

本文链接： https://blog.csdn.net/u012762410/article/details/129614919

版权

专栏06-Deep Learning 专栏收录该内容

4 篇文章 1 订阅

订阅专栏

在这里插入图片描述

CLIP简介

git地址： Chinese-CLIP
模型下载链接：模型规模 & 下载链接
作者知乎解读：
中文CLIP模型卷土重来，这次加量不加价！

CLIP使用（及改写）

Inference（需要cn_clip包）

# 计算emb函数
import torch
import numpy as np
from PIL import Image
import torchvision

import cn_clip
import cn_clip.clip as clip
from cn_clip.clip import load_from_name, available_models, load
print("Available models:", available_models())
device = "cuda" if torch.cuda.is_available() else "cpu"
print(device)
# model, preprocess = load_from_name("ViT-B-16", device=device, download_root='./pt_models')
model, preprocess = load_from_name("ViT-H-14", device=device, download_root='../02_CLIP/pt_models')  # 
model.eval()
print('finished')


from PIL import Image
from io import BytesIO
from urllib import request
def url2pil(img_url):
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
    headers = {'user-agent': user_agent}
    req = request.Request(url=img_url, headers=headers)
    response = request.urlopen(req, timeout=30)
    img= Image.open(BytesIO(response.read())).convert('RGB')
    return img

def get_img_emb(url):
    if not url: return ('1'+'***'+'url')
    try:
        img = url2pil(url)
    except:
        return ('2'+'***'+url)
    image = url2pil(url)
    with torch.no_grad():
        image = preprocess(image).unsqueeze(0).to(device)
        image_features = model.encode_image(image)
        image_features /= image_features.norm(dim=-1, keepdim=True)
        image_features = image_features.cpu().numpy()
        emb = ','.join([str(x) for x in image_features.flatten()])
        return "***".join(('0',url, emb))
        
#get_img_emb(url)


def get_txt_emb_batch(text):
    text = clip.tokenize(text).to(device)
    with torch.no_grad():
        text_features = model.encode_text(text)
    text_features = text_features.detach()
    text_features /= text_features.norm(dim=-1, keepdim=True)
    text_features = text_features.cpu().numpy()
    return text_features