Skip to content

3DSRBench和SiteBenchImage两个数据集评测的bug麻烦修复下,修复方式文中给出。 #1485

@yourlovedu

Description

@yourlovedu

评测过程中发现两个bug,麻烦有空帮忙修正下:
1.3DSRBench评测name_str未定义。
vlmeval/dataset/image_mcq.py文件中2576行的name_str应该得和其他的一样加上类似下面的代码

        name_str_map = {'chatgpt-0125': 'openai', 'gpt-4-0125': 'gpt4'}
        name_str = name_str_map[model] if model in name_str_map else model

2.SiteBenchImage读图时会有没扩展名的文件。
vlmeval/smp/file.py文件378行的parse_file函数应该参考https://github.com/EvolvingLMMs-Lab/VLMEvalKit/blob/EASI/vlmeval/smp/file.py
修改为如下

def _mime_from_pillow(p):
    from PIL import Image
    try:
        with Image.open(p) as im:
            fmt = (im.format or '').lower()
        return {
            'jpeg': 'image/jpeg', 'jpg': 'image/jpeg',
            'png': 'image/png',
            'webp': 'image/webp',
            'bmp': 'image/bmp',
            'gif': 'image/gif',
            'tiff': 'image/tiff', 'tif': 'image/tiff',
        }.get(fmt, 'unknown')
    except Exception:
        return 'unknown'


def parse_file(s):
    if osp.exists(s) and s != '.':
        assert osp.isfile(s)
        suffix = osp.splitext(s)[1].lower()
        # 添加对webp的支持
        if suffix == '.webp':
            return ('image/webp', s)
        mime = mimetypes.types_map.get(suffix, 'unknown')
        if mime == 'unknown':
            mime = _mime_from_pillow(s)
        return (mime, s)
    elif s.startswith('data:image/'):
        # To be compatible with OPENAI base64 format
        content = s[11:]
        mime = content.split(';')[0]
        content = ';'.join(content.split(';')[1:])
        dname = osp.join(LMUDataRoot(), 'files')
        assert content.startswith('base64,')
        b64 = content[7:]
        os.makedirs(dname, exist_ok=True)
        tgt = osp.join(dname, md5(b64) + '.png')
        decode_base64_to_image_file(b64, tgt)
        return parse_file(tgt)
    elif validators.url(s):
        suffix = osp.splitext(s)[1].lower()
        # 添加对webp的支持
        if suffix == '.webp':
            mime = 'image/webp'
        elif suffix in mimetypes.types_map:
            mime = mimetypes.types_map[suffix]
            dname = osp.join(LMUDataRoot(), 'files')
            os.makedirs(dname, exist_ok=True)
            tgt = osp.join(dname, md5(s) + suffix)
            download_file(s, tgt)
            return (mime, tgt)
        else:
            return ('url', s)

    else:
        return (None, s)

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions