| ... | ... | @@ -130,6 +130,9 @@ detail: 详情信息 |
|
|
|
|
|
|
|
## 实际爬虫结果的数据结构
|
|
|
|
<!--可能与超级数据一致，可能不同的data_type的爬虫结果结构不同，超级数据是把所有data_type的结果组合在一起-->
|
|
|
|
|
|
|
|
> 2021-10-29之前数据，字段 `changes_hk`，已进行清洗
|
|
|
|
|
|
|
|
```json
|
|
|
|
{
|
|
|
|
"data":
|
| ... | ... | @@ -197,17 +200,50 @@ detail: 详情信息 |
|
|
|
"data_type": "detail",
|
|
|
|
"spider_start_time": "2021-09-10 17:10:21.044",
|
|
|
|
"spider_end_time": "2021-09-10 17:10:22",
|
|
|
|
"task_params":
|
|
|
|
{
|
|
|
|
"CRNo": 446
|
|
|
|
},
|
|
|
|
"metadata":
|
|
|
|
{},
|
|
|
|
"task_params": {"CRNo": 446},
|
|
|
|
"metadata": {},
|
|
|
|
"spider_name": "enterprise_hk_spider",
|
|
|
|
"spider_ip": "10.8.1.14"
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
> 2021-10-29之后数据，爬虫仅保留压缩后的html不进行清洗，字段 `changes_hk_ele`
|
|
|
|
|
|
|
|
```json
|
|
|
|
{
|
|
|
|
"data":
|
|
|
|
{
|
|
|
|
"company_code": "1268978",
|
|
|
|
"company_type": "私人股份有限公司",
|
|
|
|
"establish_date": "2008-08-29",
|
|
|
|
"company_status": "已告解散",
|
|
|
|
"remark": "已告解散(被除名)",
|
|
|
|
"liquidation_mode": "",
|
|
|
|
"cancel_date": "2012-12-14",
|
|
|
|
"mortgage": "",
|
|
|
|
"imp_matters": "",
|
|
|
|
"company_name": "加拿大亞特蘭格國際農化有限公司",
|
|
|
|
"en_name": "CANADA YATELANGE INT'L FARMING CHEMICAL LIMITED",
|
|
|
|
"changes_hk_ele": "H4sIAMvTe2EC/7MpSUzKSVUoz0wpybBVsrRUVVJIzkksLrZVSkksSVSy4+VSgAGbkiKFpPTk/Jz8IlslZTc3Z2dLS5ACTohkBopGheKSypxUWyWwwVaGRgYFJUp2Nkl2Nmn5eSUwlcWJuamJxUn5KZVKdk8n9D5fvuH5lPnPpnY8m7702Zz5NvogtXY2+kBd+iUZOG3KSM1MzyixVTIyBLkGqAqfNU/27n8+ZQXEMhwWgEzQLymCmAVBYEFkAiTICSagOsA0QhSmEo6AgWfHZVOSolCUX15ckJgHdC5aSBsZGFg83bnF4tmcDiNLYAgA3ZBiB9KBosoGpBkmlJxfmQf0m5Kds6Ofo4ujQqRjiKuPo5+7q4KnX4i6j4KbY5Cvp5+7grOHq6+ns6OPgo+nr2eIq4uNPsgUO7ANXGC/wp1HhGVPuxY8697/dMnyJ7vmPe/c+WLG2mcL9jyd0/1y1uwX+zY97Zn2bE7ny5kTnrauedq/A9MqLi5IAIPSHTiMIUkMQcMTnAJCBgQBZD3nAKsCAAA="
|
|
|
|
},
|
|
|
|
"http_code": 200,
|
|
|
|
"error_msg": "",
|
|
|
|
"task_result": 1000,
|
|
|
|
"data_type": "detail",
|
|
|
|
"spider_start_time": "2021-10-29 18:58:18.405",
|
|
|
|
"spider_end_time": "2021-10-29 18:58:19",
|
|
|
|
"task_params": {"CRNo": 1268978},
|
|
|
|
"metadata": {},
|
|
|
|
"spider_name": "enterprise_hk",
|
|
|
|
"spider_ip": "10.8.1.58"
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
> 上面该条数据对应的html代码如下
|
|
|
|
```html
|
|
|
|
<table width="99%" class="data">\r\n <tr bgcolor="#FFCC99">\r\n\t <th class="data" style="width:120pt"><b><font class="sameasbody">名称生效日期</font></b></th>\r\n\t <th class="data" height="21">\r\n\r\n\t<b><font class="sameasbody">使用名称</font></b></th>\r\n\t \r\n\t</tr>\r\n\r\n\t\r\n\t\r\n\t \r\n\t \r\n\t \r\n\t \t\r\n\t\t\t\r\n\t\t \r\n\t\t \t\r\n\t\t\t\r\n\t\t\r\n\t \r\n\t\r\n\t\r\n\t\r\n\t<tr>\n<td rowspan="2" class="data">2008年8月29日</td><td class="data"><span class="coyname">CANADA YATELANGE INT\'L FARMING CHEMICAL LIMITED</span></td>\n</tr><tr>\n<td class="data"><span class="coyname">加拿大亞特蘭格國際農化有限公司</span></td>\n</tr>\n\n\r\n\t</table>\r\n\r\n\r\n \r\n\r\n \r\n \r\n \r\n\r\n\r\n\r\n
|
|
|
|
```
|
|
|
|
|
|
|
|
## 爬虫运行环境
|
|
|
|
<!--udm模块？scrapy？或其他-->
|
|
|
|
```buildoutcfg
|
| ... | ... | @@ -220,7 +256,7 @@ scrapy |
|
|
|
```buildoutcfg
|
|
|
|
target: node_76,
|
|
|
|
spider_name: enterprise_hk_spider
|
|
|
|
10个进程
|
|
|
|
6个进程
|
|
|
|
```
|
|
|
|
|
|
|
|
|
| ... | ... | |
