Skip to content

GitLab

crontab_bak · Changes

Page history
add: 10.8.6.27 crontab bak authored by 袁波's avatar 袁波
Show whitespace changes
Inline Side-by-side
Showing with 159 additions and 1 deletion
deploy_doc/crontab_bak.md
View page @ 06ff52fb
......@@ -117,7 +117,7 @@
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config YN
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config XZ
10 10 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config QH```
```
### 10.8.6.47
```
####
......@@ -297,3 +297,161 @@
# 欠税公告周任务,每周三提交任务
0 0 * * 3 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tax_arrears_announcement/jobs/data_pump/data_pump.py pump-data -c tax_arrears_announcement.yml
```
# 10.8.6.27
```
# >>> qcc >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
## IC失败非1000码处理
### es到sql
20 23 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_retry_task.yml
### sql至redis任务
#0 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_retry_task.yml --default_config tasks
### 临时sql补数据任务
#*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_retry_task.yml --default_config temp_tasks
*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_retry_task.yml --default_config temp_tasks_err
### 补err_pl产生的fid数据
#0 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config err_fid
### 个人详情任务（不工作时保持注释状态）
### tb_err_pl_all表
0 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config errpl_all_less_10
10 */2 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config errpl_all_login
10 */2 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config errpl_all_login2
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config fid_v3_errpl_all
## 定时vps重新拨号代理至redis(对登录爬虫影响)
50 */3 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/udms/spider_test_consumer/init_tool.py >>/home/collie/tmp/init_vps.log 2>&1
#48 */2 * * * sh /home/collie/sh/reload_qcc_login.sh stop
#52 */2 * * * sh /home/collie/sh/reload_qcc_login.sh start
### changed
0 */2 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_task.yml --default_config changed_fid_pc
0 */4 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_task.yml --default_config changed_fid_search
0 */2 * * * sh /home/collie/sh/mgr.sh task qcc id
45 * * * * sh /home/collie/sh/mgr.sh task qcc detail_person
45 * * * * sh /home/collie/sh/mgr.sh task qcc detail_person2
0 */3 * * * sh /home/collie/sh/mgr.sh task tyc id
45 */2 * * * sh /home/collie/sh/mgr.sh task tyc detail_person
# <<< qcc <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
# >>> general_taxpayer >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# <<< general_taxpayer <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
# 卫健委爬虫关停
55 0 * * * sh /home/collie/sh/reload_health.sh stop
5 5 * * * sh /home/collie/sh/reload_health.sh start
# 建筑企业资质证书
0 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_certifications/data_pump/cert.yml --default_config file
# <<< 工商和年报，公告相关任务 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# 工商总局公告爬虫任务
00 11 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_zongju_notice/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_zongju_notice/data_pump/add_spider_task.yml
00 11 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_zongju_notice/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_zongju_notice/data_pump/add_spider_task.yml --default_config product
# 年报任务提交从侦测表
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/ar_task_sql_taskhub.yml --default_config mysql_ar_2021_to_rds
# */10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/ar_task_sql_taskhub.yml --default_config mysql_ar_2021_to_rds_tmp
# 个体户年报侦测和变更：22 from es
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/es_company_to_task.yaml --default_config product > /dev/null
# 工商例行任务
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task1 > /dev/null
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task2 > /dev/null
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task3 > /dev/null
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task4 > /dev/null
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task5 > /dev/null
# 各省工商任务
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/sichuan/push_task.yaml --default_config task_sichuan
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/sichuan/push_task.yaml --default_config task_zhejiang
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/sichuan/push_task.yaml --default_config task_henan
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/sichuan/push_task.yaml --default_config task_beijing
# 工商水滴年报侦测上线从文件
#*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_online_task.yaml --default_config product_shuidi_ar
# 工商行政区可维度任务来源qxb
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/allow_task_sql_to_taskhub.yaml --default_config product
# 工商总局重启
#00 13 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#30 13 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#00 14 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#30 14 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#00 15 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#30 15 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#00 16 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#30 16 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
# 经营异常任务提交，一次性 20220301开始
# */5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/task_sql_to_taskhub.yaml --default_config product
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_company_to_task.yaml --default_config product
# 吊注销企业例行
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/es_to_task.yml --default_config product > /dev/null
# 个体户例行更新
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/es_to_task.yml --default_config product_magor1 > /dev/null
# 工商上线一次性任务
#*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_online_task.yaml --default_config product_pc
# 工商年报上线一次性任务
#*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_ar_task_sql_taskhub.yml --default_config mysql_ar_2021_to_rds_one_time
# 五一循环任务
#*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_online_task.yaml --default_config product_ks
#*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_online_task.yaml --default_config product_ks_credit_china
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/task_sql_to_taskhub.yaml --default_config product > /dev/null
# 700万2021成立的缺少2021年报和联系方式一次性侦测任务
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/es_to_task.yml --default_config product_es_700w_2021_ar_task > /dev/null
# <<< 工商和年报，公告相关任务 -----------------------------------------------------------------------
# <<< 其他项 <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
# 钉钉企典-在营企业
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_zongju_notice/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_dingdingqidian/data_pump/qidian.yml --default_config product_all_company
# 法院公告庭审直播网站补全
*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_tingshen
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_tingshen_text
# 招投标数据日常任务提交
30 8 * * * bash /home/collie/product/app_bid/deploy/data_pump_search_tasks_submit.sh `date -d '1 days ago' +\%Y-\%m-\%d` 0 0
30 17 * * * bash /home/collie/product/app_bid/deploy/data_pump_search_tasks_submit.sh `date +\%Y-\%m-\%d` 0 0
# 信用中国任务从es
*/4 * * * * cd /home/collie/product/app_credit_china/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config /home/collie/product/app_credit_china/data_pump/push_task.yml > /dev/null 2>&1
# 信用中国例行更新任务
5 * * * * cd /home/collie/product/app_credit_china/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config /home/collie/product/app_credit_china/data_pump/push_task_from_mysql.yml --default_config product
# 香港企业
## 没前缀
0 20 * * 1-6 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_enterprise/data_pump/enterprise_hk.yml
## 带符号前缀
0 20 * * 1-6 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_enterprise/data_pump/enterprise_hk.yml --default_config prefix
```
\ No newline at end of file