vn.py量化社区
By Traders, For Traders.
Member
avatar
加入于:
帖子: 123
声望: 23
@lru_cache(maxsize=10)
def load_bar_data(
    symbol: str,
    exchange: Exchange,
    interval: Interval,
    start: datetime,
    end: datetime
):
    """bar数据缓存为pkl格式到本地硬盘"""
    dir_path = f"H:\\pickle_data\\"
    file_name = f"{symbol}_{exchange.value}_{start.date()}_{end.date()}_bar"
    pickle_path = dir_path + file_name + ".pkl"
    data_size  =0 
    if not os.path.exists(pickle_path):
        bar_data = database_manager.load_bar_data( symbol, exchange, interval, start, end )
        pickle_file = open(pickle_path,'wb')    
        pickle.dump(bar_data,pickle_file)
        pickle_file.close()
    else:        
        pickle_file = open(pickle_path,'rb')
        bar_data =pickle.load(pickle_file)
        pickle_file.close()
    #pickle_data文件夹大于50G清空缓存数据
    for dirpath, dirnames, filenames in os.walk(dir_path):
        for file_name in filenames:         #当前目录所有文件名
            data_size += os.path.getsize(dirpath + file_name)
    if data_size / (1024 ** 3) > 50:
        for dirpath, dirnames, filenames in os.walk(dir_path):
            for file_name in filenames:           
                os.remove(dirpath + file_name)    
    return bar_data


@lru_cache(maxsize=10)
def load_tick_data(
    symbol: str,
    exchange: Exchange,
    start: datetime,
    end: datetime
):
    """tick数据缓存为pkl格式到本地硬盘"""
    dir_path = f"H:\\pickle_data\\"
    file_name = f"{symbol}_{exchange.value}_{start.date()}_{end.date()}_tick"
    pickle_path = dir_path + file_name + ".pkl"
    data_size  =0 
    if not os.path.exists(pickle_path):
        tick_data = database_manager.load_tick_data( symbol, exchange, start, end )
        pickle_file = open(pickle_path,'wb')    
        pickle.dump(tick_data,pickle_file)
        pickle_file.close()
    else:        
        pickle_file = open(pickle_path,'rb')
        tick_data =pickle.load(pickle_file)
        pickle_file.close()
    #pickle_data文件夹大于50G清空缓存数据
    for dirpath, dirnames, filenames in os.walk(dir_path):
        for file_name in filenames:         #当前目录所有文件名
            data_size += os.path.getsize(dirpath + file_name)
    if data_size / (1024 ** 3) > 50:
        for dirpath, dirnames, filenames in os.walk(dir_path):
            for file_name in filenames:           
                os.remove(dirpath + file_name)    
    return tick_data
Member
avatar
加入于:
帖子: 123
声望: 23

要是有更快更方便的方法欢迎大家show code

Administrator
avatar
加入于:
帖子: 3043
声望: 137

适合硬盘够大的土豪,给你加个精

Member
avatar
加入于:
帖子: 17
声望: 0

哈哈哈, 这么大的硬盘啊

Member
avatar
加入于:
帖子: 4
声望: 0

成熟的商业化方案:

  1. mysql持久化存储。
  2. redis存入基础行情数据。
  3. 时证券基础信息 pickle序列保存本地。
  4. 历史长期数据 用bcolz磁盘存储。
  5. 实时行情放在内存
Member
avatar
加入于:
帖子: 16
声望: 0

试了一下,很给力,时间节省了一大半,赞楼主

© 2015-2019 上海韦纳软件科技有限公司
备案服务号:沪ICP备18006526号-3