思路 js抓取仓库地址 py读取文件批量clone 即可 有手就行由于本地已认证过 所以不需要再py脚本配置账号密码importosimportsubprocess from pathlibimportPath from concurrent.futuresimportThreadPoolExecutor, as_completedimporttimefrom tqdmimporttqdm# 配置参数URLS_FILEgiturls.txtTARGET_DIR.\\test-front# 指定克隆到的目标目录TIMEOUT300MAX_WORKERS3# 并发数量MAX_RETRIES2# 失败重试次数# 可选Git 认证信息GIT_USERNAMEGIT_PASSWORDdef clone_single_repo(url, target_path, idx, total):克隆单个仓库 repo_nameurl.rstrip(/).rstrip(.git).split(/)[-1]repo_pathtarget_path / repo_name result_info{name:repo_name,url:url,status:unknown,message:}# 检查是否已存在ifrepo_path.exists(): result_info[status]skippedresult_info[message]目录已存在returnresult_info# 添加认证信息如果提供clone_urlurlifGIT_USERNAME and GIT_PASSWORD: clone_urlurl.replace(https://, fhttps://{GIT_USERNAME}:{GIT_PASSWORD})# 重试机制forattemptinrange(MAX_RETRIES 1): try:# 在目标目录下克隆resultsubprocess.run([git,clone, clone_url],capture_outputTrue,textTrue,timeoutTIMEOUT,cwdstr(target_path)# 在指定目录下执行)ifresult.returncode0: result_info[status]successresult_info[message]成功returnresult_info else: error_msgresult.stderr.strip()ifattemptMAX_RETRIES: time.sleep(2)else: result_info[status]failedresult_info[message]error_msg except subprocess.TimeoutExpired:ifattemptMAX_RETRIES: time.sleep(2)else: result_info[status]failedresult_info[message]f超时{TIMEOUT}秒except Exception as e: result_info[status]failedresult_info[message]str(e)breakreturnresult_info def clone_repositories():批量克隆 Git 仓库# 创建目标目录target_pathPath(TARGET_DIR)target_path.mkdir(parentsTrue,exist_okTrue)# 读取 URLtry: with open(URLS_FILE,r,encodingutf-8)as f: urls[line.strip()forlineinfifline.strip()]except FileNotFoundError: print(f❌ 错误找不到文件 {URLS_FILE})returnifnot urls: print(❌ 错误文件为空)returnprint(f 共找到 {len(urls)} 个仓库)print(f 目标目录{target_path.absolute()})print(f 并发数量{MAX_WORKERS})print(-*60)# 统计results{success:0,failed:0,skipped:0}failed_repos[]start_timetime.time()# 使用 tqdm 创建进度条with tqdm(totallen(urls),desc 克隆进度,unitrepo)as pbar: with ThreadPoolExecutor(max_workersMAX_WORKERS)as executor: futures{executor.submit(clone_single_repo, url, target_path, idx, len(urls)):(idx, url)foridx, urlinenumerate(urls,1)}forfutureinas_completed(futures): idx, urlfutures[future]resultfuture.result()results[result[status]]1# 更新进度条描述status_icon{success:✅,failed:❌,skipped:⚠️}pbar.set_postfix_str(f{status_icon.get(result[status], ?)} {result[name]})pbar.update(1)ifresult[status]failed:failed_repos.append(result)elapsed_timetime.time()- start_time# 输出统计print(\n*60)print( 克隆完成统计)print(*60)print(f✅ 成功{results[success]})print(f⚠️ 跳过{results[skipped]})print(f❌ 失败{results[failed]})print(f⏱️ 耗时{elapsed_time:.2f} 秒)# 保存失败列表iffailed_repos: print(\n*60)print(❌ 失败的仓库)print(*60)failed_filefailed_repos.txtwith open(failed_file,w,encodingutf-8)as f:forrepoinfailed_repos: print(f\n仓库{repo[name]})print(f错误{repo[message]})f.write(f{repo[url]}\n)print(f\n 失败的 URL 已保存到{failed_file})if__name____main__:print( 开始批量克隆 Git 仓库...)clone_repositories()print(\n✨ 全部完成)爬取脚本(asyncfunctionextractAllGitUrls(){const allUrlsnew Set();functionextractCurrentPage(){const linksdocument.querySelectorAll(.project-row-contents a[data-testidgroup-name]);links.forEach(link{const hreflink.getAttribute(href);if(!href)return;const fullUrlnew URL(href, window.location.origin).href;allUrls.add(fullUrl .git);});returnlinks.length;}// 轮询等待直到列表内容变化或者超时 asyncfunctionwaitForListChange(prevFirstHref, maxWait15000, interval300){const startDate.now();while(Date.now()- startmaxWait){await new Promise(rsetTimeout(r, interval));const firstLinkdocument.querySelector(.project-row-contents a[data-testidgroup-name]);const currentFirstHreffirstLink ? firstLink.getAttribute(href):null;if(currentFirstHrefcurrentFirstHref!prevFirstHref){// 内容变了再多等一小会确保渲染完整 await new Promise(rsetTimeout(r,500));returntrue;}}console.warn(等待超时内容可能未更新继续尝试抓取当前页面...);returnfalse;}functiongetNextButton(){returndocument.querySelector(li.page-item:not(.disabled) .next-page-item);}letpageCount1;while(true){console.log(正在抓取第 ${pageCount}页...);// 记录当前页第一条数据的 href用于后续对比判断是否已翻页成功 const firstLinkBeforedocument.querySelector(.project-row-contents a[data-testidgroup-name]);const prevFirstHreffirstLinkBefore ? firstLinkBefore.getAttribute(href):null;const countextractCurrentPage();console.log(本页抓取到 ${count}条);const nextBtngetNextButton();if(!nextBtn){console.log(没有更多分页了抓取结束。);break;}nextBtn.click();await waitForListChange(prevFirstHref);pageCount;if(pageCount200){console.warn(已超过 200 页强制停止。);break;}}const resultArray.from(allUrls);console.log(\n共抓取到 ${result.length}个 Git 仓库地址);console.log(result.join(\n));returnresult;})();