data-service/BondHistData.py

# -*- coding:utf-8 -*-

__author__ = 'weijie'

from EmQuantAPI import *
import datetime
import time
import traceback
from StockPojo import StockPojo
from StockHistPojo import StockHistPojo
from dbOperation import dbOperation
import pandas as pd
import json
import array
import db_config

# 获取债券历史记录
class BondHistData:

    def toGet(self):
        try:
            db = dbOperation(db_config.db_zhaiquan)
            stockPojoList = db.query_list("select id, code, name, list_date listDate from t_stock_base_info where 1=1")
            getDataTime = time.strftime('%Y-%m-%d',time.localtime(time.time()))
            for stockPojo in stockPojoList:
                if(stockPojo['listDate'] == '待上市'):
                    stockPojo['listDate'] = '1991-01-01'
                if stockPojo['listDate'] > getDataTime:
                    continue
                isAdd = False
                sql = "select dates dates from data_hist_" + stockPojo['code'].replace(".", "_").lower() + " order by realTime desc limit 1"
                dates = db.query_one(sql)
                startDate = str(stockPojo['listDate'])
                if dates is not None :
                    startDate = str(dates['dates'].replace("/", "-"))
                    dd = datetime.datetime.strptime(startDate, "%Y-%m-%d")
                    dd = (dd + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
                    startDate = dd
                stockHistPojoList = []
                try :
                    data = c.csd(
                        str(stockPojo['code']), 
                        "OPEN,CLOSE,HIGH,LOW,PRECLOSE,AVERAGE,CHANGE,PCTCHANGE,VOLUME,AMOUNT", 
                        startDate, 
                        getDataTime, 
                        "Period=1,adjustflag=1,curtype=1,Ispandas=1")
                    print("返回数据：" + str(data))
                    data.reset_index(inplace=True)
                    # print(data)
                    jsonData = data.to_json()
                    text = json.loads(jsonData)
                    for i in range(len(text['CODES'])):
                        isAdd = True
                        stockHistPojo = StockHistPojo()
                        stockHistPojo.codes = text['CODES'][str(i)].replace(".", "_")
                        stockHistPojo.open = text['OPEN'][str(i)]
                        stockHistPojo.close = text['CLOSE'][str(i)]
                        stockHistPojo.high = text['HIGH'][str(i)]
                        stockHistPojo.low = text['LOW'][str(i)]
                        stockHistPojo.preclose = text['PRECLOSE'][str(i)]
                        if "None".strip() == str(text['AVERAGE'][str(i)]).strip():
                            continue
                        stockHistPojo.average = text['AVERAGE'][str(i)]
                        stockHistPojo.change = text['CHANGE'][str(i)]
                        stockHistPojo.pctchange = text['PCTCHANGE'][str(i)]
                        stockHistPojo.volume = text['VOLUME'][str(i)]
                        stockHistPojo.highlimit = "0"
                        stockHistPojo.amount = text['AMOUNT'][str(i)]
                        stockHistPojo.turn = "0"
                        stockHistPojo.tradestatus = "0"
                        stockHistPojo.lowlimit = "0"
                        stockHistPojo.amplitude = "0"
                        stockHistPojo.tnum = "0"
                        stockHistPojo.tafactor = "0"
                        stockHistPojo.fronttafactor = "0"
                        stockHistPojo.isststock = "0"
                        stockHistPojo.isxststock = "0"
                        stockHistPojo.dates = text['DATES'][str(i)]

                        datTime = time.mktime(time.strptime(stockHistPojo.dates, "%Y/%m/%d"))
                        year = time.localtime(datTime).tm_year # 获取年份
                        yearDatTime = time.mktime(time.strptime(str(year) + "/01/01", "%Y/%m/%d"))

                        month = time.localtime(datTime).tm_mon # 获取月份
                        monthDatTime = ""
                        if 10 > month:
                            datTime = time.mktime(time.strptime(str(year) + "/0" + str(month) + "/01", "%Y/%m/%d"))
                            monthDatTime = int(datTime)
                        else:
                            datTime = time.mktime(time.strptime(str(year) + "/" + str(month) + "/01", "%Y/%m/%d"))
                            monthDatTime = int(datTime)

                        weekDatTime = time.mktime(time.strptime(stockHistPojo.dates, "%Y/%m/%d"))
                        wday = time.localtime(weekDatTime).tm_wday # 获取周
                        dt = datetime.datetime.strptime(stockHistPojo.dates, "%Y/%m/%d")
                        out_date = (dt + datetime.timedelta(days=-wday)).strftime("%Y/%m/%d")
                        weekDatTime1 = time.mktime(time.strptime(out_date, "%Y/%m/%d"))

                        #获取日
                        dayDatTime = time.mktime(time.strptime(stockHistPojo.dates, "%Y/%m/%d"))

                        stockHistPojo.dateYear = str(int(yearDatTime))
                        stockHistPojo.dateMonth = str(int(monthDatTime))
                        stockHistPojo.dateWeek = str(int(weekDatTime1))
                        stockHistPojo.dateDay = str(int(dayDatTime))
                        
                        stockHistPojoList.append(stockHistPojo)
                except Exception as ee:
                    print("数据解析错误")
                i = 0
                key = "`realTime`,`dates`,`open`,`close`,`high`,`low`,`preclose`,`average`,`change`,`pctchange`,`volume`,`highlimit`,`amount`,`turn`,`tradestatus`,`lowlimit`,`amplitude`,`tnum`,`tafactor`,`fronttafactor`,`isststock`,`isxststock`,`date_year`,`date_month`,`date_week`"
                if len(stockHistPojoList) == 0:
                    continue
                while True :
                    vlues = ""
                    # print(i)
                    # 如果数组数量大于500 截取前500个
                    if len(stockHistPojoList) - i > 500 :
                        # stockPojoList[i : 500] 等于java subList(i, 500)
                        for stockHistPojo in stockHistPojoList[i :  500 + i]:
                            vlues = vlues + "('"+str(stockHistPojo.dateDay)+"','"+str(stockHistPojo.dates)+"','" + str(stockHistPojo.open) + "','" + str(stockHistPojo.close) + "','" + str(stockHistPojo.high)+ "','" + str(stockHistPojo.low) + "','" + str(stockHistPojo.preclose) + "','" + str(stockHistPojo.average) + "','" + str(stockHistPojo.change) + "','" + str(stockHistPojo.pctchange) + "','" + str(stockHistPojo.volume) + "','" + str(stockHistPojo.highlimit) + "','" + str(stockHistPojo.amount) + "','" + str(stockHistPojo.turn) + "','" + str(stockHistPojo.tradestatus) + "','" + str(stockHistPojo.lowlimit) + "','" + str(stockHistPojo.amplitude) + "','" + str(stockHistPojo.tnum) + "','" + str(stockHistPojo.tafactor) + "','" + str(stockHistPojo.fronttafactor) + "','" + str(stockHistPojo.isststock) + "','" + str(stockHistPojo.isxststock) + "','" + str(stockHistPojo.dateYear) + "','" + str(stockHistPojo.dateMonth) + "','" + str(stockHistPojo.dateWeek) + "'),"
                        vlues = vlues[:-1]
                        db.batchInsert("data_hist_" + stockPojo['code'].replace(".", "_").lower(), key, vlues)   
                        i = i + 500
                    else :
                        # 如果数量小于500 并且不为数量不大于标记变量
                        if len(stockHistPojoList) > i :
                            for stockHistPojo in stockHistPojoList[i : len(stockHistPojoList)]:
                                vlues = vlues + "('"+str(stockHistPojo.dateDay)+"','"+str(stockHistPojo.dates)+"','" + str(stockHistPojo.open) + "','" + str(stockHistPojo.close) + "','" + str(stockHistPojo.high)+ "','" + str(stockHistPojo.low) + "','" + str(stockHistPojo.preclose) + "','" + str(stockHistPojo.average) + "','" + str(stockHistPojo.change) + "','" + str(stockHistPojo.pctchange) + "','" + str(stockHistPojo.volume) + "','" + str(stockHistPojo.highlimit) + "','" + str(stockHistPojo.amount) + "','" + str(stockHistPojo.turn) + "','" + str(stockHistPojo.tradestatus) + "','" + str(stockHistPojo.lowlimit) + "','" + str(stockHistPojo.amplitude) + "','" + str(stockHistPojo.tnum) + "','" + str(stockHistPojo.tafactor) + "','" + str(stockHistPojo.fronttafactor) + "','" + str(stockHistPojo.isststock) + "','" + str(stockHistPojo.isxststock) + "','" + str(stockHistPojo.dateYear) + "','" + str(stockHistPojo.dateMonth) + "','" + str(stockHistPojo.dateWeek) + "'),"
                            vlues = vlues[:-1]
                            db.batchInsert("data_hist_" + stockPojo['code'].replace(".", "_").lower(), key, vlues)   
                        break
                if isAdd:
                    db.update("update t_stock_base_info set is_hist_add = 1 where id = " + str(stockPojo['id']))
            db.close()

        except Exception as ee:
            print("error >>>",ee)
            traceback.print_exc()
        else:
            print("获取债券历史结束")