NetWatch数据采集器(浏览器扩展程序)配置说明

455次阅读

NetWatch浏览器扩展通过监听浏览器的网络的方式监听接口数据，旨在模拟真实用户采集目标网站的数据、开始采集前需先正常登录（若需要登录）目标网站；
NetWatch不支持采集非接口调用的后端渲染页面（这类页面的爬虫采集程序已经很多了，不是本程序的产品定位）；
采集器运行后关闭浏览器会立即结束采集任务。

插件下载安装地址：开发中，敬请期待。。。

NetWatch的配置文件是标准json文件，如下内容便是一个完整的配置文件：

// 配置项默认值及说明

{

“interval”: {“min”:1000, “max”:2000},     // 触发器执行的间隔时间范围，基于此范围生成随机数

“sleep”: {“total”: 1000, “sleep”: 3600000}, // 爬取total条数据后休眠sleep秒

“procedures”: [                      // 爬取数据的过程，通过此字段可自定义request的执行过程

{

“name”: “getQrCode”                 // 方法名(request.name)

},

{

“name”: “getGroups”

}

],

“apiToken”: “{Your http api token}”, // 后端接口的token，会作为参数追加到请求中

“requests”: [

{

“name”: “getGroups”,

“filter”: {

“path”: “/app/index.php”,

“query”: { “s”: “Home/index/index2022”}

},

“trigger”: {

“type”: “scroll”,

“selector”: “html”

},

“handler”: {

“action”: “callHttpApi”,

“type”: “http”,

“method”: “post”,

“url”: “http://127.0.0.1/wp-json/ss_app/netWatch”,

“data”: {

“data”: “@list@”      // 读取目标接口响应数据的值，@符号之间是标准的js变量取值表达式，

}

}

},

{

“name”: “getQrCode”,

“filter”: {

“path”: “/app/index.php”,

“query”: { “s”: “Home/index/huanqun” }

},

“trigger”: {

“type”: “click”,

“each”:”append”,

“selector”: “body > uni-app > uni-page > uni-page-wrapper > uni-page-body > uni-view > uni-view:nth-child(8) > uni-view.margin-top > uni-view:nth-child(1) > uni-view:nth-child(1) > uni-view > uni-view > uni-view.flex.flex-sub.justify-between.solid-bottom.padding-tb-sm.margin-left > uni-view:nth-child(2) > uni-view > uni-view > uni-button”

},

“handler”: {

“type”: “callHttpApi”,

“method”: “patch”,

“url”: “http://127.0.0.1/wp-json/ss_app/netWatch”,

“data”: {

“id”: “@id@”,

“qrcode”: {

“valueType”: “scan”,

“selector”: “body > uni-app > uni-page > uni-page-wrapper > uni-page-body > uni-view > uni-view:nth-child(8) > uni-view.margin-top > uni-view:nth-child(3) > uni-view > uni-view > uni-view.padding-sm > uni-view.u-icon.u-icon–right > uni-image > img”

}

}

},

“before”: {

“type”: “click”,

“selector”: “body > uni-app > uni-page > uni-page-wrapper > uni-page-body > uni-view > uni-view:nth-child(8) > uni-view.margin-top > uni-view:nth-child(1) > uni-view:nth-child(2) > uni-view.cu-modal.show > uni-view > uni-view.cu-bar.bg-white.justify-end > uni-view.action > uni-text”

}

}

]

}