Dev/rebase 0.14.0 (#847)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: root <root@hk01dgx039.cm.cluster> Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Signed-off-by: Taichang Zhou <tzhouam@connect.ust.hk> Signed-off-by: TangPeng <85704592@qq.com> Signed-off-by: XU Mingshi <91017482+mxuax@users.noreply.github.com> Signed-off-by: mxuax <mxuax@connect.ust.hk> Signed-off-by: Sihyeon Jang <sihyeon.jang@navercorp.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: iwzbi <wzbi@zju.edu.cn> Signed-off-by: ZeldaHuang <hzm414167@alibaba-inc.com> Signed-off-by: Yuhan Liu <30294295+liuyuhanalex@users.noreply.github.com> Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> Signed-off-by: samithuang <285365963@qq.com> Signed-off-by: linyueqian <linyueqian@outlook.com> Signed-off-by: David Chen <530634352@qq.com> Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com> Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Signed-off-by: wangyu31577 <wangyu31577@hundsun.com> Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: John Liu BUAA <liukecheng97@gmail.com> Signed-off-by: Dinesh G <G.Dinesh@ibm.com> Signed-off-by: gDINESH13 <dinesh13g@gmail.com> Signed-off-by: root <root@hk01dgx028.cm.cluster> Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk> Co-authored-by: JaredforReal <w13431838023@gmail.com> Co-authored-by: root <root@hk01dgx039.cm.cluster> Co-authored-by: JustQJ <37905360+JustQJ@users.noreply.github.com> Co-authored-by: XU Mingshi <91017482+mxuax@users.noreply.github.com> Co-authored-by: Sihyeon Jang <uneedsihyeon@gmail.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com> Co-authored-by: catcat <108673086+iwzbi@users.noreply.github.com> Co-authored-by: Ziming Huang <hzm414167@alibaba-inc.com> Co-authored-by: Yuhan Liu <30294295+liuyuhanalex@users.noreply.github.com> Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com> Co-authored-by: Samit <285365963@qq.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: wangyu31577 <wangyu31577@hundsun.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: John Liu BUAA <liukecheng97@gmail.com> Co-authored-by: D!NE$H <67671800+gDINESH13@users.noreply.github.com> Co-authored-by: root <root@hk01dgx028.cm.cluster>
This commit is contained in:
64 files changed
+3359
-906
No files matched your search
+27
@@ -305,6 +305,26 @@ def get_multi_audios_query(custom_prompt: str | None = None):
|
||||
}
|
||||
|
||||
|
||||
def get_use_audio_in_video_query(
|
||||
video_path: str | None = None,
|
||||
audio_path: str | None = None,
|
||||
custom_prompt: str | None = None,
|
||||
):
|
||||
question = custom_prompt or (
|
||||
"Describe the content of the video in details, then convert what the baby say into text."
|
||||
)
|
||||
video_url = get_video_url_from_path(video_path)
|
||||
audio_url = get_audio_url_from_path(audio_path)
|
||||
return {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video_url", "video_url": {"url": video_url}},
|
||||
{"type": "audio_url", "audio_url": {"url": audio_url}},
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
query_map = {
|
||||
"text": get_text_query,
|
||||
"use_audio": get_audio_query,
|
||||
@@ -312,6 +332,7 @@ query_map = {
|
||||
"use_video": get_video_query,
|
||||
"use_mixed_modalities": get_mixed_modalities_query,
|
||||
"use_multi_audios": get_multi_audios_query,
|
||||
"use_audio_in_video": get_use_audio_in_video_query,
|
||||
}
|
||||
|
||||
|
||||
@@ -372,6 +393,12 @@ def run_multimodal_generation(args) -> None:
|
||||
prompt = query_func(audio_path=audio_path, custom_prompt=custom_prompt)
|
||||
elif args.query_type == "text":
|
||||
prompt = query_func(custom_prompt=custom_prompt)
|
||||
elif args.query_type == "use_audio_in_video":
|
||||
prompt = query_func(
|
||||
video_path=video_path,
|
||||
audio_path=audio_path,
|
||||
custom_prompt=custom_prompt,
|
||||
)
|
||||
else:
|
||||
prompt = query_func()
|
||||
|
||||
|
||||
Reference in new issue
Block a user