Elasticsearch

# Kibana is served by a back end server. This setting specifies the port to use.
#server.port: 5601

# The URLs of the Elasticsearch instances to use for all your queries.
# elasticsearch.hosts: ["http://localhost:9200"]
 
 # If your Elasticsearch is protected with basic authentication, these settings provide
# the username and password that the Kibana server uses to perform maintenance on the Kibana
# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
# is proxied through the Kibana server.
# elasticsearch.username: "elastic"
# elasticsearch.password: "pass"

# Kibana is served by a back end server. This setting specifies the port to use.
#server.port: 5601

# The URLs of the Elasticsearch instances to use for all your queries.
 elasticsearch.hosts: "http://172.18.20.7:9200"
 
 # If your Elasticsearch is protected with basic authentication, these settings provide
# the username and password that the Kibana server uses to perform maintenance on the Kibana
# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
# is proxied through the Kibana server.
 elasticsearch.username: "elastic"
 elasticsearch.password: "123456"

PUT test_index

GET test_index/_search

GET/POST test_index/_search
{
  "query": {
    "match_all": { }
  },
  "sort": [
    {
      "datetime": "desc" # 根据datetime降序排列
    }
  ]
}

GET test_index/_mapping

POST test_index/_mapping
{
	"properties":{
		"url":{
			"type":"text"
		}
	}
}

POST test_index/_update_by_query
{
  "query": {
    "match_all": {}
  },
  "script": {
    "source": "ctx._source.url = params.last",
    "lang": "painless",
    "params": {
      "last":"http://tianya.com"
    }
  }
}

pip install elasticsearch==7.14.0

import json
import os
from elasticsearch import Elasticsearch, helpers
from time import time

def generateESInfo(info, esIndex):
    return {
        "_index": esIndex,
        # _id的值如果不写，上传时系统会自动生成
        "_source": {
            "title": info['title'],
            "content": info['content'],
            "recordDate": info['recordDate'],
            "createDate": int(time()),
            "status": [0],
            "source": info['source'],
            "url": info['url'],
            "originData": info
        }
    }


def importLocalData(dataFile):
    esClient = Elasticsearch() # 不写参数默认连接本地ES服务
    # 连接远端
    # esClient = Elasticsearch(hosts=[{"host": "172.18.20.7", "port": "9200"}],http_auth=('elastic', '123456'))
    esIndex = "myindex" # 索引名称
    batchSize = 200

    lineIndex = 0
    actions = []
    count = 0
    with open(dataFile, "r", encoding="utf8") as robj:
        for line in robj:
            lineIndex += 1
            info = json.loads(line.strip())
            actions.append(generateESInfo(info, esIndex))
            if len(actions) == batchSize:
                try:
                    helpers.bulk(esClient, actions)
                    actions = []
                    count += batchSize
                    print(count)
                except Exception as e:
                    print(e)
                    print(count)
                    return
    if len(actions) > 0:
        count += len(actions)
        helpers.bulk(esClient, actions)
        print(count)

importLocalData(os.path.join(os.path.abspath("."), "forum_data2.json")) # 将本地文件forum_data2.json上传至ES