Streaming

Data streaming is based on the Apache Storm, subscribes data from Redis, and after processing save back to Redis or Postgresql.

Reference:

• Redis Spout https://github.com/stormprocessor/storm-redis-pubsub
• Redis Bolt https://github.com/apache/storm/tree/master/external/storm-redis

Configuration

License

Arxan Fintech license.

connection kafka server consumer data

zookeeper info :

• zookeeper host :192.168.199.3:2181,192.168.199.3:2182,192.168.199.3:2183
• kafka in zookeeper root dir : /kafka

kafka info

• kafka host : 192.168.199.3:9092 #postgresql info
• postgresql host : 192.168.199.3:5432

storm spout input data output Bolt

storm bolt

pg data struct

TABLE: p2p_statistical_info

description: 各p2p公司的综合统计信息(按天统计总量) ============================ ========== ============= ====================================== column type priority description ============================ ========== ============= ====================================== p2p_code str p2p代码 p2p_name str p2p名称 total_investors int this week 投资总人次(截至该日期未关闭的投资人总量)
total_scatteredinvest_borrowers int
total_financing_borrowers int
ave_return_rate_three_month float this week 平均收益率-三月期(投资人收益,期限种类可增加)
ave_return_rate_half_year float this week 平均收益率-六月期(投资人收益,期限种类可增加) ave_return_rate_one_year float this week 平均收益率-一年期(投资人收益,期限种类可增加) total_financing int this week 理财计划总数
total_transfer int this week 债权转让总数
total_scatteredInvest int this week 散标总数

scatteredInvest_transaction int this week 散标交易次数 ?
transfer_transaction int this week 债权转让次数 ?
financing_transaction int this week 理财计划交易次数 ?
scatteredInvest_amount float this week 散标交易金额 ?
transfer_amount float this week 债权转让金额 ?
financing_amount float this week 理财计划交易金额 ?

total_borrowing_amount float N/A 还款总额(?) ave_full_term float N/A 平均满标时间(？) overdue_rate float this week 逾期率
overdue_amount float N/A 逾期金额(?)
overdue_days float N/A 平均逾期天数(?) created_at int this week 创建时间
loan_balance float N/A 贷款项目余额(?) loan_amount float this week 贷款发放总额
fullBD_percentage float this week 满标占比
leverage float N/A 资金杠杆(?) ============================ ========== ============= ======================================

CREATE TABLE  p2p_statistical_info (
  p2p_code VARCHAR (50),
  p2p_name VARCHAR (50),
  total_investors DECIMAL (12, 0),
  total_scatteredinvest_borrowers DECIMAL (12, 0),
  total_financing_borrowers DECIMAL (12, 0),
  ave_return_rate_three_month DECIMAL (12, 2),
  ave_return_rate_half_year DECIMAL (12, 2),
  ave_return_rate_one_year DECIMAL (12, 2),
  total_financing               DECIMAL (12, 0),
  total_transfer                DECIMAL (12, 0),
  total_scatteredInvest         DECIMAL (12, 0),
  scatteredInvest_transaction DECIMAL (12, 0),
  transfer_transaction DECIMAL (12, 0),
  financing_transaction DECIMAL (12, 0),
  scatteredInvest_amount DECIMAL (12, 2),
  transfer_amount DECIMAL (12, 2),
  financing_amount DECIMAL (12, 2),
  total_borrowing_amount DECIMAL (12, 2),
  ave_full_term DECIMAL (12, 2),
  overdue_rate DECIMAL (12, 2),
  overdue_amount DECIMAL (12, 2),
  overdue_days DECIMAL (12, 2),
  created_at INT,
  createdate INT,
  loan_balance DECIMAL (12, 2),
  loan_amount DECIMAL (12, 2),
  fullBD_percentage DECIMAL (12, 2),
  leverage DECIMAL (12, 2)
);

#TABLE: p2p_transaction_statistical_info

description: 各p2p公司按交易类型(平台手续费，放款，投资，平台服务费等)的统计信息(按天统计总量) ============================ ========== ============= ====================================== column type priority description ============================ ========== ============= ====================================== p2p_code str p2p代码 p2p_name str p2p名称 created_at int next week 创建时间 loanMoney decimal
investmentMoney decimal riskMoney decimal serviceMoney decimal counterFee decimal repaymentMoney decimal rechargeMoney decimal withdrawalsMoney decimal otherMoney decimal ============================ ========== ============= ====================================== create table p2p_transaction_statistical_info( p2p_code varchar(50), p2p_name varchar(50), created_at int, createDate int, loanMoney DECIMAL (12, 2), investmentMoney DECIMAL (12, 2), riskMoney DECIMAL (12, 2), serviceMoney DECIMAL (12, 2), counterFee DECIMAL (12, 2), repaymentMoney DECIMAL (12, 2), rechargeMoney DECIMAL (12, 2), withdrawalsMoney DECIMAL (12, 2), otherMoney DECIMAL (12, 2)

);

#TABLE: p2p_funds_concentration

description: 各p2p公司的资金集中度列表(按天统计) ============================ ========== ============= =================================== column type priority description ============================ ========== ============= =================================== p2p_code str p2p代码 p2p_name str p2p名称 borrow_idcard str next week 用户身份证 borrowamt DECIMAL next week 发布借款金额 actualborrowmoney DECIMAL 实际借款总额 tender_count int nexk week 借款标的数 created_at int next week 创建时间 ============================ ========== ============= ===================================

create table p2p_funds_concentration( p2p_code varchar(50), p2p_name varchar(50), borrow_idcard varchar(500), borrowamt DECIMAL (12, 2), actualborrowmoney DECIMAL (12, 2), tender_count DECIMAL (10, 0), created_at int );

#TABLE: p2p_multiloan_list description: 多头借贷列表(按天统计) ============================ ========== ============= =================================== column type priority description ============================ ========== ============= =================================== borrow_idcard str next week 用户标识 p2p_count int next week p2p个数 loan_count int next week 借贷次数 borrowamt float next week 借款金额 created_at int next week 创建时间 ============================ ========== ============= ===================================

create table p2p_multiloan_list( borrow_idcard varchar(500), p2p_count DECIMAL (10, 0), loan_count DECIMAL (10, 0), borrowamt DECIMAL (10, 2), created_at int );