学院的网站建设的意义,网络规划设计师大纲,北外网院网站建设作业,齐齐哈尔哪里做网站1. 前言
限于作者能力水平#xff0c;本文可能存在谬误#xff0c;因此而给读者带来的损失#xff0c;作者不做任何承诺。
2. 分析背景
本文分析基于 linux-4.14.132 内核代码分析#xff0c;运行环境 Ubuntu 16.04.4 LTS QEMU ARM vexpress-a9 #xff0c;rootfs 基…1. 前言
限于作者能力水平本文可能存在谬误因此而给读者带来的损失作者不做任何承诺。
2. 分析背景
本文分析基于 linux-4.14.132 内核代码分析运行环境 Ubuntu 16.04.4 LTS QEMU ARM vexpress-a9 rootfs 基于 ubuntu-base-16.04-core-armhf.tar.gz 制作。
3. soft lockup 机制
3.1 什么是 soft lockup ?
soft lockup 是指某个进程一直占住当前 CPU 其它进程无法在当前 CPU 上得到调度的情形。soft lockup 在不同的抢占模式下有着不同的情形。
3.2 各种抢占配置下的 soft lockup
3.2.1 CONFIG_PREEMPT_NONE 配置下的 soft lockup
CONFIG_PREEMPT_NONE 不支持内核态抢占致力于提高吞吐量通常用于服务器。 。在 CONFIG_PREEMPT_NONE 配置下包含死循环的进程可导致 soft lockup 。
3.2.2 CONFIG_PREEMPT 配置下的 soft lockup
CONFIG_PREEMPT 支持内核态抢占用于低延迟的桌面系统。在CONFIG_PREEMPT 配置下较长时间禁用抢占的进程可导致 soft lockup 。
3.2.3 CONFIG_PREEMPT_VOLUNTARY 配置下的 soft lockup
CONFIG_PREEMPT_VOLUNTARY 不支持内核态抢占通常用于桌面系统。相对于配置 CONFIG_PREEMPT_NONE 的情形在一些可能导致睡眠的代码路径上插入了一些调度点以降低延迟。
/** include/linux/kernel.h*/...#ifdef CONFIG_PREEMPT_VOLUNTARY
extern int _cond_resched(void);
# define might_resched() _cond_resched()
#else
# define might_resched() do { } while (0)
#endif#ifdef CONFIG_DEBUG_ATOMIC_SLEEPvoid ___might_sleep(const char *file, int line, int preempt_offset);void __might_sleep(const char *file, int line, int preempt_offset);
/*** might_sleep - annotation for functions that can sleep** this macro will print a stack trace if it is executed in an atomic* context (spinlock, irq-handler, ...).** This is a useful debugging help to be able to catch problems early and not* be bitten later when the calling function happens to sleep when it is not* supposed to.*/
# define might_sleep() \do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0)
# define sched_annotate_sleep() (current-task_state_change 0)
#elsestatic inline void ___might_sleep(const char *file, int line,int preempt_offset) { }static inline void __might_sleep(const char *file, int line,int preempt_offset) { }
# define might_sleep() do { might_resched(); } while (0)
# define sched_annotate_sleep() do { } while (0)
#endif#define might_sleep_if(cond) do { if (cond) might_sleep(); } while (0)...在 CONFIG_PREEMPT_VOLUNTARY 配置下包含死循环的进程可导致 soft lockup 。
3.3 soft lockup 的实现
3.3.1 创建 soft lockup watchdog 每 CPU 线程
为所有的 CPU 创建每 CPU 的 watchdog 内核线程该内核线程每次被调度时更新一次 watchdog时间戳同时启动一个定时器定时器到期时对比 watchdog时间戳 和 当前时间戳如果当前时间戳 - watchdog时间戳 设定的soft lockup阈值表示当前 CPU 上的当前进程已经占住 CPU 较长时间如此报告一个 soft lockup 问题 。
void __init lockup_detector_init(void)
{...cpumask_copy(watchdog_cpumask, cpu_possible_mask);...lockup_detector_setup();
}static struct smp_hotplug_thread watchdog_threads {.store softlockup_watchdog,.thread_should_run watchdog_should_run,.thread_fn watchdog,.thread_comm watchdog/%u,.setup watchdog_enable,.cleanup watchdog_cleanup,.park watchdog_disable,.unpark watchdog_enable,
};static __init void lockup_detector_setup(void)
{...lockup_detector_update_enable();.../** 创建每 CPU 的 watchdog 内核线程* . 负责更新每 CPU 的 watchdog 时间戳 * . 每次更新 watchdog 时间戳的同时启动一个定时器* 该定时器检测 watchdog 时间戳更新的时间间隔如* 果更新间隔超过设定的阈值则报告 soft lockup 问题。*/ret smpboot_register_percpu_thread_cpumask(watchdog_threads,watchdog_allowed_mask);if (ret) {pr_err(Failed to initialize soft lockup detector threads\n);return;}mutex_lock(watchdog_mutex);softlockup_threads_initialized true;lockup_detector_reconfigure();mutex_unlock(watchdog_mutex);
}static void lockup_detector_update_enable(void)
{watchdog_enabled 0;...if (soft_watchdog_user_enabled)watchdog_enabled | SOFT_WATCHDOG_ENABLED;
}int smpboot_register_percpu_thread_cpumask(struct smp_hotplug_thread *plug_thread,const struct cpumask *cpumask)
{...for_each_online_cpu(cpu) {ret __smpboot_create_thread(plug_thread, cpu); /* 创建 cpu 的内核线程 */...if (cpumask_test_cpu(cpu, cpumask))smpboot_unpark_thread(plug_thread, cpu); /* 启动 cpu 的内核线程进入 smpboot_thread_fn() */}...
}static int
__smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
{struct task_struct *tsk *per_cpu_ptr(ht-store, cpu);struct smpboot_thread_data *td;td kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu));td-cpu cpu;td-ht ht;/* * SMP 内核线程的公共入口为 smpboot_thread_fn() * 通过 smpboot_thread_fn() 调用具体的线程入口。*/tsk kthread_create_on_cpu(smpboot_thread_fn, td, cpu,ht-thread_comm);kthread_park(tsk); /* 暂停内核线程 */get_task_struct(tsk);*per_cpu_ptr(ht-store, cpu) tsk;...return 0;
}3.3.2 运行 soft lockup watchdog 内核线程
static int smpboot_thread_fn(void *data)
{struct smpboot_thread_data *td data;struct smp_hotplug_thread *ht td-ht; /* watchdog_threads */while (1) {set_current_state(TASK_INTERRUPTIBLE);preempt_disable();if (kthread_should_stop()) {__set_current_state(TASK_RUNNING);preempt_enable();/* cleanup must mirror setup */if (ht-cleanup td-status ! HP_THREAD_NONE)ht-cleanup(td-cpu, cpu_online(td-cpu));kfree(td);return 0;}if (kthread_should_park()) {__set_current_state(TASK_RUNNING);preempt_enable();if (ht-park td-status HP_THREAD_ACTIVE) {BUG_ON(td-cpu ! smp_processor_id());ht-park(td-cpu);td-status HP_THREAD_PARKED;}kthread_parkme();/* We might have been woken for stop */continue;}BUG_ON(td-cpu ! smp_processor_id());/* Check for state change setup */switch (td-status) {case HP_THREAD_NONE:__set_current_state(TASK_RUNNING);preempt_enable();if (ht-setup)ht-setup(td-cpu); /* watchdog_enable(): 初始化 watchdog时间戳以及时间戳更新间隔检测定时器 */td-status HP_THREAD_ACTIVE;continue;case HP_THREAD_PARKED:__set_current_state(TASK_RUNNING);preempt_enable();if (ht-unpark)ht-unpark(td-cpu);td-status HP_THREAD_ACTIVE;continue;}if (!ht-thread_should_run(td-cpu)) {preempt_enable_no_resched();schedule();} else {__set_current_state(TASK_RUNNING);preempt_enable();/* 更新当前 CPU 的 watchdog时间戳 */ht-thread_fn(td-cpu); /* kernel/watchdog.c: watchdog() */}}
}static void watchdog_enable(unsigned int cpu)
{/* 启动时间戳更新间隔检测定时器 */hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);hrtimer-function watchdog_timer_fn;hrtimer_start(hrtimer, ns_to_ktime(sample_period),HRTIMER_MODE_REL_PINNED);__touch_watchdog(); /* 初始 watchdog 的时间戳 */watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1); /* 设置为 RT 调度类的 SCHED_FIFO */
}static void watchdog(unsigned int cpu)
{__this_cpu_write(soft_lockup_hrtimer_cnt,__this_cpu_read(hrtimer_interrupts));__touch_watchdog(); /* watchdog 线程负责时不时的更新当前CPU的watchdog时间戳 */
}3.3.3 触发 soft lockup 问题报告
定时器到期后进入 watchdog_timer_fn() 触发可能的 soft lockup 问题报告
/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{unsigned long touch_ts __this_cpu_read(watchdog_touch_ts); /* 读取当前CPU最近的更新的时间戳 */.../* kick the softlockup detector */wake_up_process(__this_cpu_read(softlockup_watchdog)); /* 唤醒当前CPU时间戳更新线程 *//* .. and repeat */hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));if (touch_ts 0) { /* 休眠重启等类似情形下时间戳复位为0 */...__touch_watchdog(); /* 重新初始化时间戳 */return HRTIMER_RESTART; /* 重启定时器 */}duration is_softlockup(touch_ts); /* 上次更新的时间戳和当前时间戳的差值 */if (unlikely(duration)) { /* 时间戳差值大于设定的阈值 */.../* 报告 sotf lockup 问题 */pr_emerg(BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n,smp_processor_id(), duration,current-comm, task_pid_nr(current));__this_cpu_write(softlockup_task_ptr_saved, current);print_modules();print_irqtrace_events(current);if (regs)show_regs(regs);elsedump_stack();...add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);if (softlockup_panic)panic(softlockup: hung tasks);__this_cpu_write(soft_watchdog_warn, true);} else__this_cpu_write(soft_watchdog_warn, false);return HRTIMER_RESTART;
}static int is_softlockup(unsigned long touch_ts)
{unsigned long now get_timestamp();if ((watchdog_enabled SOFT_WATCHDOG_ENABLED) watchdog_thresh){/* Warn about unreasonable delays. *//* 更新时间戳的间隔超过了设定的阈值 */if (time_after(now, touch_ts get_softlockup_thresh()))return now - touch_ts;}return 0;
}4. softlock up 举例
#include linux/module.h
#include linux/sched.h
#include linux/kthread.h
#include linux/delay.hstatic struct task_struct *softlockup_task;static int softlockup_task_fn(void *ignored)
{int ret 0;while (!kthread_should_stop()) {#if (defined(CONFIG_PREEMPT_NONE) || defined(CONFIG_PREEMPT_VOLUNTARY))asm(nop);#else /* CONFIG_PREEMPT */preempt_disable();mdelay(30 * 1000);preempt_enable();#endif}return ret;
}static int __init softlockup_task_demo_init(void)
{int ret 0;softlockup_task kthread_run(softlockup_task_fn, NULL, softlockup_task);if (IS_ERR(softlockup_task)) {ret PTR_ERR(softlockup_task);printk(KERN_ERR %s: Failed to create kernel thread, ret [%d]\n, __func__, ret);}printk(KERN_INFO soft lockup task example module loaded.\n);return ret;
}static void __exit softlockup_task_demo_exit(void)
{if (softlockup_task) {kthread_stop(softlockup_task);softlockup_task NULL;}printk(KERN_INFO soft lockup task example module exited.\n);
}module_init(softlockup_task_demo_init);
module_exit(softlockup_task_demo_exit);MODULE_LICENSE(GPL);这里包含配置编译的完整代码。编译测试模块
#
# 配置
## 抢占模式 3 选一
CONFIG_PREEMPT_NONEy
#CONFIG_PREEMPT_VOLUNTARYy
#CONFIG_PREEMPTyCONFIG_DEBUG_KERNELy
CONFIG_LOCKUP_DETECTORy
CONFIG_SOFTLOCKUP_DETECTORy
CONFIG_SAMPLE_SOFTLOCKUPmcd linux-4.14.132
make ARCHarm CROSS_COMPILEarm-linux-gnueabi- -j8 Ooutputsudo mount rootfs.img temp
cd linux-4.14.132
sudo make ARCHarm CROSS_COMPILEarm-linux-gnueabi- Ooutput INSTALL_MOD_PATH/path/to/temp modules_install
cd -
sudo umount temp然后用QEMU启动系统加载测试模块
sudo qemu-system-arm \-M vexpress-a9 \-smp 4 \-m 512M \-kernel /path/to/zImage \-dtb /path/to/vexpress-v2p-ca9.dtb \-nographic \-append root/dev/mmcblk0 rw rootfstypeext4 consolettyAMA0 \-sd rootfs.img# 当前位于 QEMU 模拟器系统特权模式下# modprobe softlockup_example模块加载一段时间后报 soft lockup BUG
[ 157.356865] soft lockup task example module loaded.
[ 176.383362] INFO: rcu_sched self-detected stall on CPU
[ 176.384981] 1-...: (2113 ticks this GP) idlea42/140000000000001/0 softirq2326/2326 fqs1019
[ 176.385513] (t2100 jiffies g903 c902 q636)
[ 176.387315] NMI backtrace for cpu 1
[ 176.387904] CPU: 1 PID: 939 Comm: softlockup_task Not tainted 4.14.132 #34
[ 176.387942] Hardware name: ARM-Versatile Express
[ 176.391912] [8011149c] (unwind_backtrace) from [8010c330] (show_stack0x20/0x24)
[ 176.393289] [8010c330] (show_stack) from [806ddfd8] (dump_stack0x8c/0xa0)
[ 176.393356] [806ddfd8] (dump_stack) from [806e3dc4] (nmi_cpu_backtrace0xc0/0xc4)
[ 176.393417] [806e3dc4] (nmi_cpu_backtrace) from [806e3eb0] (nmi_trigger_cpumask_backtrace0xe8/0x12c)
[ 176.393474] [806e3eb0] (nmi_trigger_cpumask_backtrace) from [8010f490] (arch_trigger_cpumask_backtrace0x20/0x24)
[ 176.393534] [8010f490] (arch_trigger_cpumask_backtrace) from [80182ea0] (rcu_dump_cpu_stacks0xac/0xd8)
[ 176.393672] [80182ea0] (rcu_dump_cpu_stacks) from [80182478] (rcu_check_callbacks0x7f8/0x9f8)
[ 176.393725] [80182478] (rcu_check_callbacks) from [80187f84] (update_process_times0x44/0x6c)
[ 176.393775] [80187f84] (update_process_times) from [80197144] (tick_periodic0x4c/0xcc)
[ 176.393828] [80197144] (tick_periodic) from [80197368] (tick_handle_periodic0x38/0x98)
[ 176.393877] [80197368] (tick_handle_periodic) from [8010ffe4] (twd_handler0x40/0x50)
[ 176.393924] [8010ffe4] (twd_handler) from [80172128] (handle_percpu_devid_irq0x98/0x24c)
[ 176.393979] [80172128] (handle_percpu_devid_irq) from [8016c728] (generic_handle_irq0x34/0x44)
[ 176.394031] [8016c728] (generic_handle_irq) from [8016cd3c] (__handle_domain_irq0x6c/0xc4)
[ 176.394125] [8016cd3c] (__handle_domain_irq) from [80101508] (gic_handle_irq0x5c/0xa0)
[ 176.394166] [80101508] (gic_handle_irq) from [8010d10c] (__irq_svc0x6c/0x90)
[ 176.394240] Exception stack(0x9e90df18 to 0x9e90df60)
[ 176.394533] df00: 00000000 9e983bc0
[ 176.394835] df20: 00000000 9e983bc0 9e983bc0 00000000 9f5fc940 9e90c000 00000000 9e983bdc
[ 176.394945] df40: 9e8fdd30 9e90df74 9e90df68 9e90df68 7f000020 7f000020 00000013 ffffffff
[ 176.396350] [8010d10c] (__irq_svc) from [7f000020] (softlockup_task_fn0x20/0x30 [softlockup_example])
[ 176.396667] [7f000020] (softlockup_task_fn [softlockup_example]) from [80143ae0] (kthread0x144/0x174)
[ 176.396718] [80143ae0] (kthread) from [80107ee8] (ret_from_fork0x14/0x2c)
[ 204.214887] watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [softlockup_task:939]
[ 204.215332] Modules linked in: softlockup_example
[ 204.215593] CPU: 1 PID: 939 Comm: softlockup_task Not tainted 4.14.132 #34
[ 204.215604] Hardware name: ARM-Versatile Express
[ 204.215648] task: 9f700c00 task.stack: 9e90c000
[ 204.215701] PC is at kthread_should_stop0x30/0x54
[ 204.215736] LR is at softlockup_task_fn0x20/0x30 [softlockup_example]
[ 204.215755] pc : [80143524] lr : [7f000020] psr: 00000013
[ 204.215770] sp : 9e90df50 ip : 9e90df68 fp : 9e90df64
[ 204.215785] r10: 9e8fdd30 r9 : 9e983bdc r8 : 00000000
[ 204.215804] r7 : 9e90c000 r6 : 9f5fc940 r5 : 00000000 r4 : 9f700c00
[ 204.215821] r3 : 00208040 r2 : 00000000 r1 : 9e983bc0 r0 : 00000000
[ 204.215875] Flags: nzcv IRQs on FIQs on Mode SVC_32 ISA ARM Segment none
[ 204.215899] Control: 10c5387d Table: 7e92806a DAC: 00000051
[ 204.215927] CPU: 1 PID: 939 Comm: softlockup_task Not tainted 4.14.132 #34
[ 204.215936] Hardware name: ARM-Versatile Express
[ 204.215979] [8011149c] (unwind_backtrace) from [8010c330] (show_stack0x20/0x24)
[ 204.216011] [8010c330] (show_stack) from [806ddfd8] (dump_stack0x8c/0xa0)
[ 204.216039] [806ddfd8] (dump_stack) from [80108acc] (show_regs0x1c/0x20)
[ 204.216067] [80108acc] (show_regs) from [801b5f08] (watchdog_timer_fn0x248/0x2c0)
[ 204.216095] [801b5f08] (watchdog_timer_fn) from [80188f48] (hrtimer_run_queues0x1b8/0x370)
[ 204.216147] [80188f48] (hrtimer_run_queues) from [80187f14] (run_local_timers0x24/0x50)
[ 204.216182] [80187f14] (run_local_timers) from [80187f7c] (update_process_times0x3c/0x6c)
[ 204.216207] [80187f7c] (update_process_times) from [80197144] (tick_periodic0x4c/0xcc)
[ 204.216232] [80197144] (tick_periodic) from [80197368] (tick_handle_periodic0x38/0x98)
[ 204.216256] [80197368] (tick_handle_periodic) from [8010ffe4] (twd_handler0x40/0x50)
[ 204.216282] [8010ffe4] (twd_handler) from [80172128] (handle_percpu_devid_irq0x98/0x24c)
[ 204.216312] [80172128] (handle_percpu_devid_irq) from [8016c728] (generic_handle_irq0x34/0x44)
[ 204.216338] [8016c728] (generic_handle_irq) from [8016cd3c] (__handle_domain_irq0x6c/0xc4)
[ 204.216363] [8016cd3c] (__handle_domain_irq) from [80101508] (gic_handle_irq0x5c/0xa0)
[ 204.216385] [80101508] (gic_handle_irq) from [8010d10c] (__irq_svc0x6c/0x90)
[ 204.216399] Exception stack(0x9e90df00 to 0x9e90df48)
[ 204.216511] df00: 00000000 9e983bc0 00000000 00208040 9f700c00 00000000 9f5fc940 9e90c000
[ 204.216617] df20: 00000000 9e983bdc 9e8fdd30 9e90df64 9e90df68 9e90df50 7f000020 80143524
[ 204.216657] df40: 00000013 ffffffff
[ 204.216697] [8010d10c] (__irq_svc) from [80143524] (kthread_should_stop0x30/0x54)
[ 204.216730] [80143524] (kthread_should_stop) from [7f000020] (softlockup_task_fn0x20/0x30 [softlockup_example])
[ 204.216766] [7f000020] (softlockup_task_fn [softlockup_example]) from [80143ae0] (kthread0x144/0x174)
[ 204.216791] [80143ae0] (kthread) from [80107ee8] (ret_from_fork0x14/0x2c)从上面可以看到soft lockup 机制是依赖于中断的当前 CPU 被占住无法调度的情形下只有中断才能打断它而用于检测的定时器接口正是在中断上下文下执行。
5. soft lockup 用户空间接口
/proc/sys/kernel/soft_watchdog # 是否启用 softlockup 功能
/proc/sys/kernel/softlockup_panic # lockup是否导致kernel panic
/proc/sys/kernel/softlockup_all_cpu_backtrace # 是否不只输出lockup CPU调用栈还输出其它CPU调用栈
/proc/sys/kernel/watchdog_thresh # softlockup 触发时间阈值
/proc/sys/kernel/watchdog_cpumask # 启用 softlockup 检测的CPU掩码
文章转载自： http://www.morning.czwed.com.gov.cn.czwed.com http://www.morning.nclbk.cn.gov.cn.nclbk.cn http://www.morning.bnmfq.cn.gov.cn.bnmfq.cn http://www.morning.qfkxj.cn.gov.cn.qfkxj.cn http://www.morning.ksgjn.cn.gov.cn.ksgjn.cn http://www.morning.ndrzq.cn.gov.cn.ndrzq.cn http://www.morning.qtbnm.cn.gov.cn.qtbnm.cn http://www.morning.rcjwl.cn.gov.cn.rcjwl.cn http://www.morning.nwczt.cn.gov.cn.nwczt.cn http://www.morning.lrjtx.cn.gov.cn.lrjtx.cn http://www.morning.nynpf.cn.gov.cn.nynpf.cn http://www.morning.ctrkh.cn.gov.cn.ctrkh.cn http://www.morning.trrhj.cn.gov.cn.trrhj.cn http://www.morning.jghty.cn.gov.cn.jghty.cn http://www.morning.hslgq.cn.gov.cn.hslgq.cn http://www.morning.gkktj.cn.gov.cn.gkktj.cn http://www.morning.ggtkk.cn.gov.cn.ggtkk.cn http://www.morning.kaylyea.com.gov.cn.kaylyea.com http://www.morning.qrsm.cn.gov.cn.qrsm.cn http://www.morning.qllcm.cn.gov.cn.qllcm.cn http://www.morning.ldqzz.cn.gov.cn.ldqzz.cn http://www.morning.rfwqt.cn.gov.cn.rfwqt.cn http://www.morning.lddpj.cn.gov.cn.lddpj.cn http://www.morning.dfkby.cn.gov.cn.dfkby.cn http://www.morning.rcklc.cn.gov.cn.rcklc.cn http://www.morning.tkchg.cn.gov.cn.tkchg.cn http://www.morning.sgbss.cn.gov.cn.sgbss.cn http://www.morning.wqjpl.cn.gov.cn.wqjpl.cn http://www.morning.lnwdh.cn.gov.cn.lnwdh.cn http://www.morning.ntwxt.cn.gov.cn.ntwxt.cn http://www.morning.jbqwb.cn.gov.cn.jbqwb.cn http://www.morning.bpmft.cn.gov.cn.bpmft.cn http://www.morning.wsgyq.cn.gov.cn.wsgyq.cn http://www.morning.wqpb.cn.gov.cn.wqpb.cn http://www.morning.jbtwq.cn.gov.cn.jbtwq.cn http://www.morning.kqzrt.cn.gov.cn.kqzrt.cn http://www.morning.fqyxb.cn.gov.cn.fqyxb.cn http://www.morning.rnzwh.cn.gov.cn.rnzwh.cn http://www.morning.sjftk.cn.gov.cn.sjftk.cn http://www.morning.hsrpc.cn.gov.cn.hsrpc.cn http://www.morning.rxpp.cn.gov.cn.rxpp.cn http://www.morning.wyjpt.cn.gov.cn.wyjpt.cn http://www.morning.mwmxs.cn.gov.cn.mwmxs.cn http://www.morning.bfcxf.cn.gov.cn.bfcxf.cn http://www.morning.qmqgx.cn.gov.cn.qmqgx.cn http://www.morning.tfwsk.cn.gov.cn.tfwsk.cn http://www.morning.pswqx.cn.gov.cn.pswqx.cn http://www.morning.mxhys.cn.gov.cn.mxhys.cn http://www.morning.qlckc.cn.gov.cn.qlckc.cn http://www.morning.rjrnx.cn.gov.cn.rjrnx.cn http://www.morning.cjsrg.cn.gov.cn.cjsrg.cn http://www.morning.rnygs.cn.gov.cn.rnygs.cn http://www.morning.tpbhf.cn.gov.cn.tpbhf.cn http://www.morning.hphfy.cn.gov.cn.hphfy.cn http://www.morning.tfsyk.cn.gov.cn.tfsyk.cn http://www.morning.brrxz.cn.gov.cn.brrxz.cn http://www.morning.kzqpn.cn.gov.cn.kzqpn.cn http://www.morning.lhsdf.cn.gov.cn.lhsdf.cn http://www.morning.cywf.cn.gov.cn.cywf.cn http://www.morning.wsyst.cn.gov.cn.wsyst.cn http://www.morning.smdiaosu.com.gov.cn.smdiaosu.com http://www.morning.lfqnk.cn.gov.cn.lfqnk.cn http://www.morning.ryfqj.cn.gov.cn.ryfqj.cn http://www.morning.rmppf.cn.gov.cn.rmppf.cn http://www.morning.nzkc.cn.gov.cn.nzkc.cn http://www.morning.ryxdr.cn.gov.cn.ryxdr.cn http://www.morning.nkjpl.cn.gov.cn.nkjpl.cn http://www.morning.bwmm.cn.gov.cn.bwmm.cn http://www.morning.bnpn.cn.gov.cn.bnpn.cn http://www.morning.khtyz.cn.gov.cn.khtyz.cn http://www.morning.qgzmz.cn.gov.cn.qgzmz.cn http://www.morning.kpbgp.cn.gov.cn.kpbgp.cn http://www.morning.dmkhd.cn.gov.cn.dmkhd.cn http://www.morning.qngcq.cn.gov.cn.qngcq.cn http://www.morning.swsrb.cn.gov.cn.swsrb.cn http://www.morning.cmdfh.cn.gov.cn.cmdfh.cn http://www.morning.wzknt.cn.gov.cn.wzknt.cn http://www.morning.qtwd.cn.gov.cn.qtwd.cn http://www.morning.wdykx.cn.gov.cn.wdykx.cn http://www.morning.ynjhk.cn.gov.cn.ynjhk.cn
