Hive处理小文件问题(实战篇)

set hive.exec.dynamic.partition=true;
set hive.exec.dynamic.partition.mode=nonstrict;
#限制单个MapReduce任务可产生的动态分区数量，防止资源过载。
set hive.exec.max.dynamic.partitions.pernode=100000;
set hive.exec.max.dynamic.partitions.pernode.Maximum=100000;
set hive.exec.max.dynamic.partitions=100000;
set hive.exec.max.created.files=100000;
insert overwrite table db.tablename partition (pic_type, platform)
select aa
, bb
, pic_type
, platform
from tablename
distribute by pic_type, platform;