Wednesday, July 29, 2015

ES 4 Test

  1. 2015.03.03
    1. bashrc
      1. export INNERIP=`hostname -i`
        export ES_HEAP_SIZE=8g
        export ES_CLASSPATH=/etc/hadoop/conf:/opt/cloudera/parcels/CDH-5.3.0-1.cdh5.3.0.p0.30/lib/hadoop/libexec/../../hadoop/lib/*:/opt/cloudera/parcels/CDH-5.3.0-1.cdh5.3.0.p0.30/lib/hadoop/libexec/../../hadoop/.//*:/opt/cloudera/parcels/CDH-5.3.0-1.cdh5.3.0.p0.30/lib/hadoop/libexec/../../hadoop-hdfs/./:/opt/cloudera/parcels/CDH-5.3.0-1.cdh5.3.0.p0.30/lib/hadoop/libexec/../../hadoop-hdfs/lib/*:/opt/cloudera/parcels/CDH-5.3.0-1.cdh5.3.0.p0.30/lib/hadoop/libexec/../../hadoop-hdfs/.//*:/opt/cloudera/parcels/CDH-5.3.0-1.cdh5.3.0.p0.30/lib/hadoop/libexec/../../hadoop-yarn/lib/*:/opt/cloudera/parcels/CDH-5.3.0-1.cdh5.3.0.p0.30/lib/hadoop/libexec/../../hadoop-yarn/.//*:/opt/cloudera/parcels/CDH/lib/hadoop-mapreduce/lib/*:/opt/cloudera/parcels/CDH/lib/hadoop-mapreduce/.//*
    2. configuration
      1. cluster.name: test
      2. node.name: ${HOSTNAME}
      3. transport.host: ${INNERIP}
      4. discovery.zen.ping.multicast.enabled: false
      5. discovery.zen.ping.unicast.hosts: ["10.0.2.81", "10.0.2.82", "10.0.2.83"]
      6. indices.fielddata.cache.size: 40%
  2. 2015.03.02
    1. snapshot and restore
      1. repository register
        1. PUT _snapshot/hdfs
          {
          "type": "hdfs",
          "settings": {
          "path": "/backup/elasticsearch"
          }
          }
      2. repository verification
        1. POST _snapshot/hdfs/_verify
      3. snapshot
        1. PUT _snapshot/hdfs/20150302
      4. monitoring snapshot/restore progress
        1. GET _snapshot/hdfs/20150302/_status
        2. GET _snapshot/hdfs/20150302
      5. snapshot information and status
        1. GET _snapshot/hdfs/20150302
        2. GET _snapshot/hdfs/_all 
        3. GET _snapshot/_status 
        4. GET _snapshot/hdfs/_status 
        5. GET _snapshot/hdfs/20150302/_status
      6. restore
        1. POST _snapshot/hdfs/20150302/_restore
      7. snapshot deletion / stopping currently running snapshot and restore operations
        1. DELETE _snapshot/hdfs/20150302
      8. repository deletion
        1. DELETE _snapshot/hdfs
      9. reference
        1. http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-snapshots.html
    2. rolling update
      1. Disable shard reallocation
        1. curl -XPUT localhost:9200/_cluster/settings -d '{ "transient" : { "cluster.routing.allocation.enable" : "none" } }'
      2. Shut down a single node within the cluster
        1. curl -XPOST 'http://localhost:9200/_cluster/nodes/_local/_shutdown'
      3. Confirm that all shards are correctly reallocated to the remaining running nodes
      4. Download newest version
      5. Extract the zip or tarball to a new directory
      6. Copy the configuration files from the old Elasticsearch installation’s config directory to the new Elasticsearch installation’s config directory
      7. Move data files from the old Elasticsesarch installation’s data directory
      8. Install plugins
      9. Start the now upgraded node
      10. Confirm that it joins the cluster
      11. Re-enable shard reallocation
        1. curl -XPUT localhost:9200/_cluster/settings -d '{ "transient" : { "cluster.routing.allocation.enable" : "all" } }'
      12. Observe that all shards are properly allocated on all nodes
      13. Repeat this process for all remaining nodes
      14. Reference
        1. http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup-upgrade.html#rolling-upgrades
  3. 2015.02.13
    1. MySQL Slow Query Log Mapping
      1. PUT msql-2015
{
  "mappings": {
    "log": {
      "properties": {
        "@timestamp": {
          "type": "date",
          "format": "dateOptionalTime"
        },
        "@version": {
          "type": "string"
        },
        "host": {
          "type": "string",
          "index": "not_analyzed"
        },
        "ip": {
          "type": "string",
          "index": "not_analyzed"
        },
        "lock_time": {
          "type": "double"
        },
        "message": {
          "type": "string",
          "index": "not_analyzed"
        },
        "query": {
          "type": "string"
        },
        "query_time": {
          "type": "double"
        },
        "rows_examined": {
          "type": "double"
        },
        "rows_sent": {
          "type": "double"
        },
        "type": {
          "type": "string"
        },
        "user": {
          "type": "string"
        }
      }
    }
  }
}
    2. MySQL Slow Query Dump Mapping
      1. PUT msqld-2015
{
  "mappings": {
    "dump": {
      "properties": {
        "@timestamp": {
          "type": "date",
          "format": "dateOptionalTime"
        },
        "@version": {
          "type": "string"
        },
        "count": {
          "type": "double"
        },
        "host": {
          "type": "string",
          "index": "not_analyzed"
        },
        "ip": {
          "type": "string",
          "index": "not_analyzed"
        },
        "lock": {
          "type": "double"
        },
        "message": {
          "type": "string",
          "index": "not_analyzed"
        },
        "query": {
          "type": "string"
        },
        "rows": {
          "type": "double"
        },
        "time": {
          "type": "double"
        },
        "type": {
          "type": "string"
        },
        "user": {
          "type": "string"
        }
      }
    }
  }
}
  4. 2015.02.12
    1. MySQL Slow Query Log & Dump Mappings
      1. PUT msqld-2015
{
  "mappings": {
    "log": {
      "properties": {
        "@timestamp": {
          "type": "date",
          "format": "dateOptionalTime"
        },
        "@version": {
          "type": "string"
        },
        "host": {
          "type": "string",
          "index": "not_analyzed"
        },
        "ip": {
          "type": "string",
          "index": "not_analyzed"
        },
        "lock_time": {
          "type": "double"
        },
        "message": {
          "type": "string",
          "index": "not_analyzed"
        },
        "query": {
          "type": "string"
        },
        "query_time": {
          "type": "double"
        },
        "rows_examined": {
          "type": "double"
        },
        "rows_sent": {
          "type": "double"
        },
        "type": {
          "type": "string"
        },
        "user": {
          "type": "string"
        }
      }
    },
    "dump": {
      "properties": {
        "@timestamp": {
          "type": "date",
          "format": "dateOptionalTime"
        },
        "@version": {
          "type": "string"
        },
        "count": {
          "type": "double"
        },
        "host": {
          "type": "string",
          "index": "not_analyzed"
        },
        "ip": {
          "type": "string",
          "index": "not_analyzed"
        },
        "lock": {
          "type": "double"
        },
        "message": {
          "type": "string",
          "index": "not_analyzed"
        },
        "query": {
          "type": "string"
        },
        "rows": {
          "type": "double"
        },
        "time": {
          "type": "double"
        },
        "type": {
          "type": "string"
        },
        "user": {
          "type": "string"
        }
      }
    }
  }
}
  5. 2015.01.26
    1. bashrc
      1. export ES_HEAP_SIZE=9g
  6. 2015.01.19
    1. 재시작 스크립트
      1. curl -XPUT localhost:9200/_cluster/settings -d '{"transient":{"cluster.routing.allocation.disable_allocation":true}}'
sleep 1s
curl -XPOST 'http://localhost:9200/_cluster/nodes/_local/_shutdown'
sleep 1s
bin/elasticsearch -d
sleep 3s
curl -XPUT localhost:9200/_cluster/settings -d '{"transient":{"cluster.routing.allocation.disable_allocation":false}}'
    2. 설정
      1. cluster.name: dev
node.name: "dev01"
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ["10.0.2.81", "10.0.2.82", "10.0.2.83"]
indices.fielddata.cache.size: 40%
  7. ~ 2015.01.01
    1. Settings
      1. curl -XPUT localhost:9200/_cluster/settings -d '{"transient":{"indices.recovery.concurrent_streams":6,"indices.recovery.max_bytes_per_sec":"50mb"}}'
      2. ES_HEAP_SIZE: 50% (< 32g)
      3. indices.fielddata.cache.size: 40%
    2. Scenarios
      1. heap size setting
      2. export ES_HEAP_SIZE=9g
      3. curl -XPUT localhost:9200/_cluster/settings -d '{"transient":{"cluster.routing.allocation.disable_allocation":true}}'
      4. curl -XPOST 'http://localhost:9200/_cluster/nodes/_local/_shutdown'
      5. bin/elasticsearch -d
      6. curl -XPUT localhost:9200/_cluster/settings -d '{"transient":{"cluster.routing.allocation.disable_allocation":false}}'
      7. TBD
Posted by at

No comments:

Post a Comment

Note: Only a member of this blog may post a comment.

Subscribe to: Post Comments (Atom)