再更新一下，接之前的版本油猴脚本，抓取LeetCode题解[2]，在原来的基础上增加了单独抓取单篇题解的功能，再做这个的时候其实遇到个问题，主要难点在于如何监听浏览器地址栏的变化，另开一个文章来说《无刷新页面监听URL变化实现》
至于新增这个功能的原因如下，在使用初版的功能全量的抓取了自己的题解文章之后，对于每天新增的题解不能单独抓取补充到indexDB之中，只能再次全量抓取一下，显然不太方便，所以又改进了下增加了这样的功能
另外我试了下，这个抓取用户题解文章列表的接口是不做权限验证的，也就是说A用户可以抓取B用户的所有题解列表，我找了下使用场景，当A用户点开某个B用户的LeetCode个人主页的时候，下面有一块展示区域是防止当前B用户的题解列表的。这是一个正常使用的场景，也就是说，你可以抓取任何你喜欢的用户的所有题解列表
另外修改了一点样式。在右边单独建了小的Panel把按钮都放进去，到处分散的不大好看，
下一步内容，解析题目、题解中的图片标签，抓取图片内容并base64之后作为文本保存下来，使题解内容脱离对力扣图片资源的依赖
// ==UserScript==
// @name LeetCode My Solution Collector
// @namespace http://blog.CheungQ.me/
// @version 0.3
// @description 帮助你收集获取你在LeetCode上写的所有题解，将下面的MY_ACCOUNT 值修改为你的用户ID，最简单的查询方法，开的你LeetCode主页，地址栏URL中的最后一段就是了。例如：“https://leetcode.cn/u/张三的用户ID/”
// @author CheungQ@qq.com
// @match https://leetcode.cn/*
// @grant none
// ==/UserScript==
const MY_ACCOUNT = "张三的用户ID";
const LEETCODE_GRAPHQL = "https://leetcode.cn/graphql/";
/**
* 下载文件的
* @param value
* @param type
* @param name
*/
let doSave = (value,type,name)=>{let blob;if(typeof window.Blob=="function"){blob=new Blob([value],{type:type})}else{let BlobBuilder=window.BlobBuilder||window.MozBlobBuilder||window.WebKitBlobBuilder||window.MSBlobBuilder;let bb=new BlobBuilder();bb.append(value);blob=bb.getBlob(type)}let URL=window.URL||window.webkitURL;let blobUrl=URL.createObjectURL(blob);let anchor=document.createElement("a");if('download'in anchor){anchor.style.visibility="hidden";anchor.href=blobUrl;anchor.download=name;document.body.appendChild(anchor);let evt=document.createEvent("MouseEvents");evt.initEvent("click",true,true);anchor.dispatchEvent(evt);document.body.removeChild(anchor)}else if(navigator.msSaveBlob){navigator.msSaveBlob(blob,name)}else{location.href=blobUrl}}
/**
* indexDB封装的
*/
class DBWrapper{constructor(name,version,{onupgradeneeded,onversionchange=this._onversionchange}={}){this._name=name;this._version=version;this._onupgradeneeded=onupgradeneeded;this._onversionchange=onversionchange;this._db=null}get db(){return this._db}async open(){if(this._db)return;this._db=await new Promise((resolve,reject)=>{let openRequestTimedOut=false;setTimeout(()=>{openRequestTimedOut=true;reject(new Error("The open request was blocked and timed out"))},this.OPEN_TIMEOUT);const openRequest=indexedDB.open(this._name,this._version);openRequest.onerror=()=>reject(openRequest.error);openRequest.onupgradeneeded=evt=>{if(openRequestTimedOut){openRequest.transaction.abort();evt.target.result.close()}else if(this._onupgradeneeded){this._onupgradeneeded(evt)}};openRequest.onsuccess=({target})=>{const db=target.result;if(openRequestTimedOut){db.close()}else{db.onversionchange=this._onversionchange.bind(this);resolve(db)}}});return this}async getKey(storeName,query){return(await this.getAllKeys(storeName,query,1))[0]}async getAll(storeName,query,count){return await this.getAllMatching(storeName,{query,count})}async getAllKeys(storeName,query,count){return(await this.getAllMatching(storeName,{query,count,includeKeys:true})).map(({key})=>key)}async getAllMatching(storeName,{index,query=null,direction="next",count,includeKeys}={}){return await this.transaction([storeName],"readonly",(txn,done)=>{const store=txn.objectStore(storeName);const target=index?store.index(index):store;const results=[];target.openCursor(query,direction).onsuccess=({target})=>{const cursor=target.result;if(cursor){const{primaryKey,key,value}=cursor;results.push(includeKeys?{primaryKey,key,value}:value);if(count&&results.length>=count){done(results)}else{cursor.continue()}}else{done(results)}}})}async transaction(storeNames,type,callback){await this.open();return await new Promise((resolve,reject)=>{const txn=this._db.transaction(storeNames,type);txn.onabort=({target})=>reject(target.error);txn.oncomplete=()=>resolve();callback(txn,value=>resolve(value))})}async _call(method,storeName,type,...args){const callback=(txn,done)=>{txn.objectStore(storeName)[method](...args).onsuccess=({target})=>{done(target.result)}};return await this.transaction([storeName],type,callback)}_onversionchange(){this.close()}close(){if(this._db){this._db.close();this._db=null}}static async deleteDatabase(name){await new Promise((resolve,reject)=>{const request=indexedDB.deleteDatabase(name);request.onerror=({target})=>{reject(target.error)};request.onblocked=()=>{reject(new Error("Delete blocked"))};request.onsuccess=()=>{resolve()}})}}DBWrapper.prototype.OPEN_TIMEOUT=2000;(function(){const methodsToWrap={readonly:["get","count","getKey","getAll","getAllKeys"],readwrite:["add","put","clear","delete"]};for(const[mode,methods]of Object.entries(methodsToWrap)){for(const method of methods){if(method in IDBObjectStore.prototype){DBWrapper.prototype[method]=async function(storeName,...args){return await this._call(method,storeName,mode,...args)}}}}})();
(function(){
try{
new window.CustomEvent('T');
}catch(e){
let CustomEvent = function(event, params){
params = params || { bubbles: false, cancelable: false, detail: undefined };
let evt = document.createEvent('CustomEvent');
evt.initCustomEvent(event, params.bubbles, params.cancelable, params.detail);
return evt;
};
CustomEvent.prototype = window.Event.prototype;
window.CustomEvent = CustomEvent;
}
})();
let publishCustomEvent = async ( eventName, eventParam , domElement) => {
if (domElement === undefined){
domElement = window;
}
if ( domElement instanceof EventTarget){
let customEvent = new CustomEvent(eventName, {detail:eventParam});
domElement.dispatchEvent(customEvent);
// console.log({eventName,eventParam})
}
};
let _historyWrap = function(type) {
const orig = history[type];
return function() {
const rv = orig.apply(this, arguments);
publishCustomEvent(type,arguments)
return rv;
};
};
history.pushState = _historyWrap('pushState');
history.replaceState = _historyWrap('replaceState');
let newElement = (tagName, styles) => {
let el = document.createElement(tagName)
if (styles && styles instanceof Object){
for (let styleKey in styles) {
if (el.style.hasOwnProperty(styleKey)){
el.style[styleKey] = styles[styleKey]
}
}
}
return el
}
let createPanel = (innerElements)=>{
let div = newElement("div",{
width: '2px',
background: 'rgb(56 56 56 / 77%)',
display: 'block',
position: 'fixed',
right: '0',
top: '100px',
paddingTop: '5px',
paddingRight: '4px',
borderRadius: '4px'
});
let wrapper = newElement("div",{
width: '100%',
height: 'auto',
display: 'block',
float: 'right',
overflow: 'hidden',
})
let icon = newElement("i",{
position: 'absolute',
left: '-20px',
top: '0px',
width: '0',
height: '0',
margin: '0 auto',
border: '10px solid transparent',
borderRightColor: 'gray',
})
icon.addEventListener('click',()=>{
div.style.width = div.style.width === '2px'?'auto':'2px'
if (icon.style.borderLeftColor === 'gray'){
icon.style.borderRightColor = 'gray'
icon.style.borderLeftColor = 'transparent'
icon.style.left = '-20px'
}else{
icon.style.borderLeftColor = 'gray'
icon.style.borderRightColor = 'transparent'
icon.style.left = '-10px'
}
})
if (innerElements){
for (let innerElement of innerElements) {
wrapper.append(innerElement)
}
}
div.append(icon)
div.append(wrapper)
document.body.append(div)
return div
}
let createButton = (func, btnText, additionStyles) => {
let btn = document.createElement("button")
let styles = {
padding: '5px',
color: 'white',
background: 'rgb(0, 0, 0)',
borderRadius: '5px',
float: 'right',
clear: 'both',
width: '55px',
marginBottom: '5px',
marginLeft: '7px',
fontSize: '12px',
}
if (additionStyles){
styles = {...styles, ...additionStyles}
}
for (let styleKey in styles) {
btn.style[styleKey] = styles[styleKey]
}
btn.innerText = btnText
btn.addEventListener("click", func)
return btn;
}
let solutionPath = (path) =>{
if (typeof path === 'string' && path.substr(0,10) === '/problems/'){
let pathArr = path.split("/");
if (pathArr.length === 6 && pathArr[3] === 'solution'){
return {
problem:pathArr[2],
solution:pathArr[4]
};
}
}
return false;
}
const CQ_EVENT_TYPES = {
NEW_SOLUTION_ARTICLES:"NEW_SOLUTION_ARTICLES",
NEXT_SOLUTION_PAGE:"NEXT_SOLUTION_PAGE",
GET_SOLUTION_CONTENT:"GET_SOLUTION_CONTENT",
GET_QUESTION_CONTENT:"GET_QUESTION_CONTENT",
};
//在这里可以定义一些你不需要的字段信息的Key
const KEYS_TO_BE_DELETED = {
QUESTION:[
'content',
'companyTagStats',
'envInfo',
'codeSnippets',
'contributors',
'langToValidPlayground',
],
SOLUTION:[
// 'author',
'next',
'prev',
'question',
]
}
const regex = /!\[(.+)\]\((.+)\)/g;
(function () {
'use strict';
let dbName = 'solutionArticle', version = 1, storeName = 'mySolution'
let db = new DBWrapper(dbName,version,{
onupgradeneeded: e => {
const db = e.target.result;
const objStore = db.createObjectStore(storeName, {
// autoIncrement: true,
keyPath: 'slug'
});
objStore.createIndex("questionFrontendId", "question.questionFrontendId", { unique: !1 });
}
})
//题解详情页单独抓取
let singleSolutionBtn = createButton(() => {
let {problem,solution} = solutionPath(window.location.pathname);
getQuestionAndSolution(problem, solution)
}, "本题解")
if (!solutionPath(window.location.pathname)){
singleSolutionBtn.style.display = 'none'
}
let statusBtn = createButton(null,"Status",{background:'gray'})
let panel = createPanel([
createButton(() => doErgodic(), "列表"),
createButton(() =>
db.getAll(storeName, null,0).then((r =>{
doSave(JSON.stringify(r),'text/latex', 'solutions.json')
}))
, "下载"),
singleSolutionBtn,
statusBtn,
])
// console.log(db);
let historyChangeHandler = (e)=>{
let {detail} = e
singleSolutionBtn.style.display = 'none'
let pathInfo = solutionPath(detail[2]);
if (pathInfo === false){
return
}
singleSolutionBtn.style.display = 'inherit'
}
window.addEventListener('pushState',historyChangeHandler,true)
window.addEventListener('replaceState',historyChangeHandler,true)
window.addEventListener('popstate',()=>{
if (!solutionPath(window.location.pathname)){
singleSolutionBtn.style.display = 'none'
}else{
singleSolutionBtn.style.display = 'inherit'
}
})
let doErgodic = ()=>{
eventListenerRegister();
db.open();
solutionPageQuery(0)
// db.close();
// eventListenerUnRegister();
}
let eventListenerRegister = ()=>{
window.addEventListener(CQ_EVENT_TYPES.NEXT_SOLUTION_PAGE,ergodicNextPage,false)
window.addEventListener(CQ_EVENT_TYPES.NEW_SOLUTION_ARTICLES,getQuestionAndSolutionList,false)
}
let eventListenerUnRegister = ()=>{
window.removeEventListener(CQ_EVENT_TYPES.NEXT_SOLUTION_PAGE,ergodicNextPage)
window.removeEventListener(CQ_EVENT_TYPES.NEW_SOLUTION_ARTICLES,getQuestionAndSolutionList)
}
let ergodicNextPage = (e)=>{
let {detail: {page}} = e
if (typeof page === "number"){
solutionPageQuery(page)
}
}
let getQuestionAndSolution = (questionTitleSlug, solutionSlug) =>{
Promise.all([
getQuestion(questionTitleSlug),
getSolution(solutionSlug)
]).then(res=>{
let [question,solutionArticle] = res;
for (let key of KEYS_TO_BE_DELETED.QUESTION) {
if (question.hasOwnProperty(key)){
delete question[key]
}
}
for (let key of KEYS_TO_BE_DELETED.SOLUTION) {
if (solutionArticle.hasOwnProperty(key)){
delete solutionArticle[key]
}
}
db.put(storeName, {slug: solutionSlug,question,solutionArticle})
statusBtn.style.background = 'gray'
}).catch((errorMessage) => {
console.error(errorMessage)
statusBtn.style.background = 'gray'
})
}
let getQuestionAndSolutionList = (e) =>{
let {detail: {list}} = e
// console.log(list)
if (list instanceof Array){
for (let item of list) {
statusBtn.style.background = 'green'
let {node: {slug, question: {questionTitleSlug}}} = item;
// console.log({slug,questionTitleSlug});
getQuestionAndSolution(questionTitleSlug,slug)
}
}
}
let getQuestion = (slug) => {
let p = {
"operationName": "questionData",
"variables": {"titleSlug": slug},
"query": "query questionData($titleSlug: String!) {\n question(titleSlug: $titleSlug) {\n questionId\n questionFrontendId\n categoryTitle\n boundTopicId\n title\n titleSlug\n content\n translatedTitle\n translatedContent\n isPaidOnly\n difficulty\n likes\n dislikes\n isLiked\n similarQuestions\n contributors {\n username\n profileUrl\n avatarUrl\n __typename\n }\n langToValidPlayground\n topicTags {\n name\n slug\n translatedName\n __typename\n }\n companyTagStats\n codeSnippets {\n lang\n langSlug\n code\n __typename\n }\n stats\n hints\n solution {\n id\n canSeeDetail\n __typename\n }\n status\n sampleTestCase\n metaData\n judgerAvailable\n judgeType\n mysqlSchemas\n enableRunCode\n envInfo\n book {\n id\n bookName\n pressName\n source\n shortDescription\n fullDescription\n bookImgUrl\n pressImgUrl\n productUrl\n __typename\n }\n isSubscribed\n isDailyQuestion\n dailyRecordStatus\n editorType\n ugcQuestionId\n style\n exampleTestcases\n jsonExampleTestcases\n __typename\n }\n}\n"
}
return leetCodeRequest(p,data=>data.data.question)
}
let getSolution = (slug) =>{
let param = {
"operationName": "solutionDetailArticle",
"variables": {
"slug": slug,
"orderBy": "DEFAULT"
},
"query": "query solutionDetailArticle($slug: String!, $orderBy: SolutionArticleOrderBy!) {\n solutionArticle(slug: $slug, orderBy: $orderBy) {\n ...solutionArticle\n content\n question {\n questionTitleSlug\n __typename\n }\n position\n next {\n slug\n title\n __typename\n }\n prev {\n slug\n title\n __typename\n }\n __typename\n }\n}\n\nfragment solutionArticle on SolutionArticleNode {\n rewardEnabled\n canEditReward\n uuid\n title\n slug\n sunk\n chargeType\n status\n identifier\n canEdit\n canSee\n reactionType\n reactionsV2 {\n count\n reactionType\n __typename\n }\n tags {\n name\n nameTranslated\n slug\n tagType\n __typename\n }\n createdAt\n thumbnail\n author {\n username\n profile {\n userAvatar\n userSlug\n realName\n __typename\n }\n __typename\n }\n summary\n topic {\n id\n commentCount\n viewCount\n __typename\n }\n byLeetcode\n isMyFavorite\n isMostPopular\n isEditorsPick\n hitCount\n videosInfo {\n videoId\n coverUrl\n duration\n __typename\n }\n __typename\n}\n"
}
return leetCodeRequest(param,data=>data.data.solutionArticle )
}
let solutionPageQuery = (page) => {
let pageSize = 50;
let param = {
"operationName": "solutionArticles",
"variables": {
"userSlug": MY_ACCOUNT,
"skip": page * pageSize,
"first": 50,
"query": ""
},
"query": "query solutionArticles($userSlug: String!, $skip: Int, $first: Int, $query: String) {\n solutionArticles(userSlug: $userSlug, skip: $skip, first: $first, query: $query) {\n totalNum\n edges {\n node {\n ...solutionArticle\n question {\n questionTitle\n questionTitleSlug\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n }\n}\n\nfragment solutionArticle on SolutionArticleNode {\n ipRegion\n rewardEnabled\n canEditReward\n uuid\n title\n slug\n sunk\n chargeType\n status\n identifier\n canEdit\n canSee\n reactionType\n reactionsV2 {\n count\n reactionType\n __typename\n }\n tags {\n name\n nameTranslated\n slug\n tagType\n __typename\n }\n createdAt\n thumbnail\n author {\n username\n profile {\n userAvatar\n userSlug\n realName\n __typename\n }\n __typename\n }\n summary\n topic {\n id\n commentCount\n viewCount\n __typename\n }\n byLeetcode\n isMyFavorite\n isMostPopular\n isEditorsPick\n hitCount\n videosInfo {\n videoId\n coverUrl\n duration\n __typename\n }\n __typename\n}\n"
}
return leetCodeRequest(param,data=>{
let {edges} = data.data.solutionArticles;
if (edges instanceof Array && edges.length > 0){
publishCustomEvent(CQ_EVENT_TYPES.NEW_SOLUTION_ARTICLES,{list:edges})
publishCustomEvent(CQ_EVENT_TYPES.NEXT_SOLUTION_PAGE,{page:page+1})
}
})
}
let leetCodeRequest = (param,callback) =>{
let options = {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify(param)
}
return new Promise((resolve, reject) => {
fetch(LEETCODE_GRAPHQL, options).then((res) => {
if (res.ok) {
//如果取数据成功
res.json().then((data) => {
resolve(callback(data))
})
} else {
console.error(res.status);
reject(res)
//查看获取状态
}
}).catch((res) => {
//输出一些错误信息
console.error(res.status);
reject(res)
})
})
}
})();
发表评论