data-service/hist/SotckHistData.py

# -*- coding:utf-8 -*-

__author__ = 'weijie'

from EmQuantAPI import *
import datetime
import time
import traceback
from StockPojo import StockPojo
from StockHistPojo import StockHistPojo
from dbOperation import dbOperation
import pandas as pd
import json
import array
import db_config

class SotckHistData:

    def toGet(self):
        try:
            # csd使用范例
            db = dbOperation(db_config.db_gupiao)
            stockPojoList = db.query_list("select id, code, name, list_date listDate from t_stock_base_info where 1 = 1")
            getDataTime = time.strftime('%Y-%m-%d',time.localtime(time.time()))
            for stockPojo in stockPojoList:
                # print(str(stockPojo['code']))
                # 先去获取历史表中最后一条 如果有就使用历史表中的最后一条时间当作起始时间去获取数据
                sql = "select dates dates from data_hist_" + stockPojo['code'].replace(".", "_").lower() + " order by realTime desc limit 1"
                dates = db.query_one(sql)
                isAdd = False
                startDate = str(stockPojo['listDate'])
                if dates is not None :
                    startDate = str(dates['dates'].replace("/", "-"))
                data = c.csd(
                    str(stockPojo['code']), 
                    "OPEN,CLOSE,HIGH,LOW,PRECLOSE,AVERAGE,CHANGE,PCTCHANGE,VOLUME,HIGHLIMIT,AMOUNT,TURN,TRADESTATUS,LOWLIMIT,AMPLITUDE,TNUM,TAFACTOR,FRONTTAFACTOR,ISSTSTOCK,ISXSTSTOCK", 
                    startDate, 
                    getDataTime, 
                    "Period=1,adjustflag=1,curtype=1,Ispandas=1")
                print(data)
                data.reset_index(inplace=True)
                jsonData = data.to_json()
                text = json.loads(jsonData)
                stockHistPojoList = []
                for i in range(len(text['CODES'])):
                    isAdd = True
                    stockHistPojo = StockHistPojo()
                    stockHistPojo.codes = text['CODES'][str(i)].replace(".", "_")
                    stockHistPojo.open = text['OPEN'][str(i)]
                    stockHistPojo.close = text['CLOSE'][str(i)]
                    stockHistPojo.high = text['HIGH'][str(i)]
                    stockHistPojo.low = text['LOW'][str(i)]
                    stockHistPojo.preclose = text['PRECLOSE'][str(i)]
                    if "None".strip() == str(text['AVERAGE'][str(i)]).strip():
                        continue
                    stockHistPojo.average = text['AVERAGE'][str(i)]
                    stockHistPojo.change = text['CHANGE'][str(i)]
                    stockHistPojo.pctchange = text['PCTCHANGE'][str(i)]
                    stockHistPojo.volume = text['VOLUME'][str(i)]
                    stockHistPojo.highlimit = text['HIGHLIMIT'][str(i)]
                    stockHistPojo.amount = text['AMOUNT'][str(i)]
                    stockHistPojo.turn = text['TURN'][str(i)]
                    stockHistPojo.tradestatus = text['TRADESTATUS'][str(i)]
                    stockHistPojo.lowlimit = text['LOWLIMIT'][str(i)]
                    stockHistPojo.amplitude = text['AMPLITUDE'][str(i)]
                    stockHistPojo.tnum = text['TNUM'][str(i)]
                    stockHistPojo.tafactor = text['TAFACTOR'][str(i)]
                    stockHistPojo.fronttafactor = text['FRONTTAFACTOR'][str(i)]
                    stockHistPojo.isststock = text['ISSTSTOCK'][str(i)]
                    stockHistPojo.isxststock = text['ISXSTSTOCK'][str(i)]
                    stockHistPojo.dates = text['DATES'][str(i)]

                    datTime = time.mktime(time.strptime(stockHistPojo.dates, "%Y/%m/%d"))
                    year = time.localtime(datTime).tm_year # 获取年份
                    yearDatTime = time.mktime(time.strptime(str(year) + "/01/01", "%Y/%m/%d"))

                    month = time.localtime(datTime).tm_mon # 获取月份
                    monthDatTime = ""
                    if 10 > month:
                        datTime = time.mktime(time.strptime(str(year) + "/0" + str(month) + "/01", "%Y/%m/%d"))
                        monthDatTime = int(datTime)
                    else:
                        datTime = time.mktime(time.strptime(str(year) + "/" + str(month) + "/01", "%Y/%m/%d"))
                        monthDatTime = int(datTime)

                    weekDatTime = time.mktime(time.strptime(stockHistPojo.dates, "%Y/%m/%d"))
                    wday = time.localtime(weekDatTime).tm_wday # 获取周
                    dt = datetime.datetime.strptime(stockHistPojo.dates, "%Y/%m/%d")
                    out_date = (dt + datetime.timedelta(days=-wday)).strftime("%Y/%m/%d")
                    weekDatTime1 = time.mktime(time.strptime(out_date, "%Y/%m/%d"))

                    #获取日
                    dayDatTime = time.mktime(time.strptime(stockHistPojo.dates, "%Y/%m/%d"))

                    stockHistPojo.dateYear = str(int(yearDatTime))
                    stockHistPojo.dateMonth = str(int(monthDatTime))
                    stockHistPojo.dateWeek = str(int(weekDatTime1))
                    stockHistPojo.dateDay = str(int(dayDatTime))
                    stockHistPojoList.append(stockHistPojo)
                i = 0
                key = "`realTime`,`dates`,`open`,`close`,`high`,`low`,`preclose`,`average`,`change`,`pctchange`,`volume`,`highlimit`,`amount`,`turn`,`tradestatus`,`lowlimit`,`amplitude`,`tnum`,`tafactor`,`fronttafactor`,`isststock`,`isxststock`,`date_year`,`date_month`,`date_week`"
                # 批量插入 每次插入500
                while True :
                    vlues = ""
                    # print(i)
                    # 如果数组数量大于500 截取前500个
                    if len(stockHistPojoList) - i > 500 :
                        # stockPojoList[i : 500] 等于java subList(i, 500)
                        for stockHistPojo in stockHistPojoList[i :  500 + i]:
                            vlues = vlues + "('"+str(stockHistPojo.dateDay)+"','"+str(stockHistPojo.dates)+"','" + str(stockHistPojo.open) + "','" + str(stockHistPojo.close) + "','" + str(stockHistPojo.high)+ "','" + str(stockHistPojo.low) + "','" + str(stockHistPojo.preclose) + "','" + str(stockHistPojo.average) + "','" + str(stockHistPojo.change) + "','" + str(stockHistPojo.pctchange) + "','" + str(stockHistPojo.volume) + "','" + str(stockHistPojo.highlimit) + "','" + str(stockHistPojo.amount) + "','" + str(stockHistPojo.turn) + "','" + str(stockHistPojo.tradestatus) + "','" + str(stockHistPojo.lowlimit) + "','" + str(stockHistPojo.amplitude) + "','" + str(stockHistPojo.tnum) + "','" + str(stockHistPojo.tafactor) + "','" + str(stockHistPojo.fronttafactor) + "','" + str(stockHistPojo.isststock) + "','" + str(stockHistPojo.isxststock) + "','" + str(stockHistPojo.dateYear) + "','" + str(stockHistPojo.dateMonth) + "','" + str(stockHistPojo.dateWeek) + "'),"
                        vlues = vlues[:-1]
                        db.batchInsert("data_hist_" + stockPojo['code'].replace(".", "_").lower(), key, vlues)   
                        i = i + 500
                    else :
                        # 如果数量小于500 并且不为数量不大于标记变量
                        if len(stockHistPojoList) > i :
                            for stockHistPojo in stockHistPojoList[i : len(stockHistPojoList)]:
                                vlues = vlues + "('"+str(stockHistPojo.dateDay)+"','"+str(stockHistPojo.dates)+"','" + str(stockHistPojo.open) + "','" + str(stockHistPojo.close) + "','" + str(stockHistPojo.high)+ "','" + str(stockHistPojo.low) + "','" + str(stockHistPojo.preclose) + "','" + str(stockHistPojo.average) + "','" + str(stockHistPojo.change) + "','" + str(stockHistPojo.pctchange) + "','" + str(stockHistPojo.volume) + "','" + str(stockHistPojo.highlimit) + "','" + str(stockHistPojo.amount) + "','" + str(stockHistPojo.turn) + "','" + str(stockHistPojo.tradestatus) + "','" + str(stockHistPojo.lowlimit) + "','" + str(stockHistPojo.amplitude) + "','" + str(stockHistPojo.tnum) + "','" + str(stockHistPojo.tafactor) + "','" + str(stockHistPojo.fronttafactor) + "','" + str(stockHistPojo.isststock) + "','" + str(stockHistPojo.isxststock) + "','" + str(stockHistPojo.dateYear) + "','" + str(stockHistPojo.dateMonth) + "','" + str(stockHistPojo.dateWeek) + "'),"
                            vlues = vlues[:-1]
                            db.batchInsert("data_hist_" + stockPojo['code'].replace(".", "_").lower(), key, vlues)   
                        break
                if isAdd:
                    db.update("update t_stock_base_info set is_hist_add = 1 where id = " + str(stockPojo['id']))
            db.close()

        except Exception as ee:
            print("error >>>",ee)
            traceback.print_exc()
        else:
            print("demo end")