Important DSE, Cassandra, and Spark configuration files.
1. /etc/default/dse
What is SPARK_ENABLED for?1. /etc/default/dse
# NOTICE: See also /etc/dse/cassandra/cassandra-env.sh # EXTRA_CLASSPATH provides the means to extend Cassandra's classpath with # additional libraries. It is formatted as a colon-delimited list of # class directories and/or jar files. For example, to enable the # JMX-to-web bridge install libmx4j-java and uncomment the following. #EXTRA_CLASSPATH="/usr/share/java/mx4j-tools.jar" # enable this to start Hadoop's JobTracker and/or TaskTrackers on this machine. # If left disabled, this will act as a regular Cassandra node. HADOOP_ENABLED=0 # enable this to set the replication factor for CFS. Note that this will only # have an effect the first time a cluster is started with HADOOP_ENABLED=1 and # after that will be a no-op. Defaults to 1. #CFS_REPLICATION_FACTOR=1 # enable this to start Solr search indexing on this machine. If left disabled, # this will act as a regular Cassandra node. SOLR_ENABLED=0 # enable thist to start Spark integration. If left disabled, this # will act as a regular Cassandra node. SPARK_ENABLED=1 # enabled this to start CFS; it is required only if you want to start CFS without # Hadoop/Spark trackers. CFS_ENABLED=0 # Install root if [ -d /usr/share/dse ]; then DSE_HOME=/usr/share/dse fi # Set this to your install location and remove the leading hashsign #DSE_HOME=your_install_location # Location of log output OUTPUT_FILE="/var/log/cassandra/output.log" # Configuration directory CONFDIR=/etc/dse/cassandra # Set the PID file location here PIDFILE=/var/run/$NAME/$NAME.pid # Where do Hadoop log files go? This will override the default #HADOOP_LOG_DIR= # Where do Tomcat log files go? This will override the default #TOMCAT_LOGS= # Ths user to use for the service CASSANDRA_USER=cassandra # Ths group to use for the service CASSANDRA_GROUP=cassandra # Spark home directory #SPARK_HOME=your_spark_install_location # Spark configuration files location SPARK_CONF_DIR=/etc/dse/spark # Shark home directory #SHARK_HOME=your_spark_install_location # Shark configuration files location SHARK_CONF_DIR=/etc/dse/shark
2. /etc/dse/dse.yaml
# Enable the Hive Meta Store via Cassandra
hive_meta_store_enabled: true
# Kerberos options
#
# The qop is the "Quality of Protection" for each connection. Used by clients
# and servers. Below is a list of valid values and their meanings.
# auth - (default) authentication only
# auth-int - authentication plus integity protection of all transmitted data
# auth-conf - authetication plus integrity protection and encryption of all
# transmitted data
# Warning: Encryption using auth-conf is separate and completely independent
# of whether encryption is done using SSL. So that if auth-conf is selected
# here and SSL is enabled, then the transmitted data will be encrypted twice.
kerberos_options:
keytab: resources/dse/conf/dse.keytab
service_principal: dse/_HOST@REALM
http_principal: HTTP/_HOST@REALM
qop: auth
# LDAP options
#
# These are options will be used if the com.datastax.bdp.cassandra.auth.LdapAuthenticator
# is configured as the authenticator in cassandra.yaml
#
ldap_options:
server_host: localhost
server_port: 389
# DN of the user that be used to search for users on the LDAP server. This user should
# only have the necessary permissions to do the search
# If not present then an anonymous bind will be used for the search
search_dn: cn=Admin
# Password of the search user
search_password: secret
# Set to true to use an SSL encrypted connection. In this case the server_port needs
# to be set to the ldaps port for the server
use_ssl: false
# Set to true to initiate a TLS encrypted connection on the default ldap port
use_tls: false
truststore_path:
truststore_password:
truststore_type: jks
user_search_base: ou=users,dc=example,dc=com
user_search_filter: (uid={0})
# Validity period for the credentials cache in milli-seconds (remote bind is an expensive
# operation). Defaults to 0, set to 0 to disable.
credentials_validity_in_ms: 0
# Validity period for the search cache in seconds. Defaults to 0, set to 0 to disable.
search_validity_in_seconds: 0
connection_pool:
max_active: 8
max_idle: 8
# To ensure that data with a TTL is purged from Solr indexes when it expires,
# DSE periodically checks indexes for data that has exceeded its TTL. These settings
# control the scheduling & execution of those checks.
ttl_index_rebuild_options:
# by default, schedule a check every 300 seconds
fixed_rate_period: 300
# the first check is delayed to speed up startup time
initial_delay: 20
# documents subject to TTL are checked in batches: this configures the max number of docs
# checked per batch
max_docs_per_batch: 200
# Solr shard transport options, for inter-node communication between Solr nodes.
shard_transport_options:
#
# Default type from 4.5.0 onwards is "netty" for TCP-based communication,
# providing lower latency, improved throughput and reduced resource consumption.
# The other type is "http" for plain old Solr communication via the standard
# HTTP-based interface.
type: netty
#
# Options specific to the "netty" transport type.
#
# The TCP listen port, mandatory if you either want to use the "netty" transport
# type, or want to later migrate to it from the "http" one. If you plan to use
# and stay with the "http" one, either comment it out or set it to -1.
netty_server_port: 8984
# The number of server acceptor threads (default is number of available processors).
# netty_server_acceptor_threads:
# The number of server worker threads (default is number of available processors * 8).
# netty_server_worker_threads:
# The number of client worker threads (default is number of available processors * 8).
# netty_client_worker_threads:
# The max number of client connections (default is 100).
# netty_client_max_connections:
# The client request timeout, in milliseconds (default is 60000).
# netty_client_request_timeout:
#
# Options specific to the "http" transport type.
#
# HTTP shard client timeouts in milliseconds.
# Default is the same as Solr, that is 0, meaning no timeout at all; it is
# strongly suggested to change it to a finite value, to avoid blocking operations.
# Notice these settings are valid across Solr cores.
# http_shard_client_conn_timeout: 0
# http_shard_client_socket_timeout: 0
# Solr indexing settings
#
# Max number of concurrent asynchronous indexing threads per Solr core.
# Default is "number of available processors" * 2; if set at 1,
# the system reverts to the synchronous behavior, where data is
# synchronously written into Cassandra and indexed by Solr.
#
# max_solr_concurrency_per_core: 2
#
# The back pressure threshold is the total number of queued asynchronous
# indexing requests per core, computed at Solr commit time;
# when exceeded, back pressure kicks in to avoid excessive
# resources consumption, causing throttling of new incoming requests.
# Default is 500.
#
# back_pressure_threshold_per_core: 500
#
# The max time to wait for flushing of async index updates, happening either
# at Solr commit time or Cassandra flush time.
# Flushing should always complete successfully, in order to fully sync Solr indexes
# with Cassandra data, so should always be set at a reasonable high value,
# expressed in minutes.
# Default is 5.
#
# flush_max_time_per_core: 5
#
# The max time to wait for each Solr core to load upon startup or create/reload operations, expressed in minutes.
# This is an advanced option, which should be changed only if any exceptions happen during core loading.
# Default is 1.
# load_max_time_per_core: 1
# Applies the configured Cassandra disk failure policy to index write failures.
# Default is disabled (false).
#
# enable_index_disk_failure_policy: false
# Solr cql query options
#
# Max number of threads to use for retrieving rows during CQL Solr queries.
# This value is cross-request and cross-core.
# Default is "number of available processors" * 10.
#
# cql_solr_query_executor_threads: 2
#
# Max time in milliseconds to wait for each row to be read from Cassandra during
# CQL Solr queries.
# Default is 10000 (10 seconds).
#
# cql_solr_query_row_timeout: 10000
# CQL performance objects features:
# * CQL Slow Log
# * CQL System Info
# * User Level Latency Tracking
# * Resource Level Latency Tracking
# * Database Summary Statistics
# CQL slow log settings
cql_slow_log_options:
enabled: false
threshold_ms: 100
ttl_seconds: 86400
async_writers: 1
# CQL system info tables settings
cql_system_info_options:
enabled: false
refresh_rate_ms: 10000
# Data Resource latency tracking settings
resource_level_latency_tracking_options:
enabled: false
refresh_rate_ms: 10000
# Database summary stats options
db_summary_stats_options:
enabled: false
refresh_rate_ms: 10000
# Cluster summary stats options
cluster_summary_stats_options:
enabled: false
refresh_rate_ms: 10000
# Column Family Histogram data tables options
histogram_data_options:
enabled: false
refresh_rate_ms: 10000
retention_count: 3
# User/Resource latency tracking settings
user_level_latency_tracking_options:
enabled: false
refresh_rate_ms: 10000
top_stats_limit: 100
quantiles: false
# Solr Performance Objects
# Solr indexing error log options
solr_indexing_error_log_options:
enabled: false
ttl_seconds: 604800
async_writers: 1
# Solr slow query log options
solr_slow_sub_query_log_options:
enabled: false
ttl_seconds: 604800
async_writers: 1
threshold_ms: 100
# Solr UpdateHandler metrics options
solr_update_handler_metrics_options:
enabled: false
ttl_seconds: 604800
refresh_rate_ms: 60000
# Solr request handler metrics options
solr_request_handler_metrics_options:
enabled: false
ttl_seconds: 604800
refresh_rate_ms: 60000
# Solr index statistics options
solr_index_stats_options:
enabled: false
ttl_seconds: 604800
refresh_rate_ms: 60000
# Solr cache statistics options
solr_cache_stats_options:
enabled: false
ttl_seconds: 604800
refresh_rate_ms: 60000
# Solr latency snapshot options
solr_latency_snapshot_options:
enabled: false
ttl_seconds: 604800
refresh_rate_ms: 60000
# The directory where system keys are kept
#
# Keys used for sstable encryption must be distributed to all nodes
# Dse will need to be able to read and write to the directory.
#
# This directory should have 700 permissions and belong to the dse user
system_key_directory: /etc/dse/conf
# If this is set to true, DSE will expect the following config values to be encrypted:
# resources/cassandra/conf/cassandra.yaml:
# server_encryption_options.keystore_password
# server_encryption_options.truststore_password
# client_encryption_options.keystore_password
# client_encryption_options.truststore_password
# resources/dse/conf/dse.yaml:
# ldap_options.search_password
# ldap_options.truststore_password
#
# it's an error if the passwords aren't encrypted
#
# config values can be encrypted with dsetool encryptconfigvalue
config_encryption_active: false
# the name of the system key used to encrypt / decrypt passwords stored
# in configuration files.
#
# If config_encryption_active is true, it's an error if a valid key with
# this name isn't in the system key directory
#
# keyfiles can be created with dsetool createsystemkey
config_encryption_key_name: system_key
# The fraction of available system resources to be used by Spark Worker
#
# This the only initial value, once it is reconfigured, the new value is stored
# and retrieved on next run.
initial_spark_worker_resources: 0.7
# Audit logging options
audit_logging_options:
enabled: false
# The logger used for logging audit information
# Available loggers are:
# CassandraAuditWriter: logs audit info to a cassandra table. This logger can be run either synchronously, or
# asynchronously. Audit logs are stored in the dse_audit.audit_log table.
# When run synchronously, a query will not execute until it has been written
# to the audit log table successfully. If there is a failure between when an audit event is
# written, and it's query is executed, the audit logs may contain queries that were never
# executed.
# Log4JAuditWriter: logs audit info to a log4j logger. The logger name is `DataAudit`, and can be configured
# in the log4j-server.properties file.
logger: Log4JAuditWriter
# Comma separated list of audit event categories to be included or excluded from the audit log.
# Categories are: QUERY, DML, DDL, DCL, AUTH, ADMIN
# Specify either included or excluded categories. Specifying both is an error
# included_categories:
# excluded_categories:
# Comma separated list of keyspaces to be included or excluded from the audit log.
# Specify either included or excluded keyspaces. Specifying both is an error
# included_keyspaces:
# excluded_keyspaces:
# The amount of time, in hours, audit events are retained by supporting loggers
# Currently, only the CassandraAuditWriter supports retention time
# values of 0 or less retain events forever
retention_time: 0
cassandra_audit_writer_options:
# sets the mode the writer runs in.
#
# When run synchonously, a query is not executed until the audit event is successfully written.
#
# When run asynchronously, audit events are queued for writing to the audit table, but are
# not neccesarily logged before the query executes. A pool of writer threads consumes the
# audit events from the queue, and writes them to the audit table in batch queries. While
# this substantially improves performance under load, if there is a failure between when
# a query is executed, and it's audit event is written to the table, the audit table may
# be missing entries for queries that were executed.
# valid options are 'sync' and 'async'
mode: sync
# The maxium number of events the writer will dequeue before writing them out to the table. If you're seeing
# warnings in your logs about batches being too large, decrease this value. Increasing batch_size_warn_threshold_in_kb
# in cassandra.yaml is also an option, but make sure you understand the implications before doing so.
#
# Only used in async mode. Must be >0
batch_size: 50
# The maximum amount of time in milliseconds an event will be dequeued by a writer before being written out. This
# prevents events from waiting too long before being written to the table when there's not a lot of queries happening.
#
# Only used in async mode. Must be >0
flush_time: 500
# The number of worker threads asynchronously logging events to the CassandraAuditWriter.
#
# Only used in async mode. Must be >0
num_writers: 10
# The size of the queue feeding the asynchronous audit log writer threads. When there are more events being
# produced than the writers can write out, the queue will fill up, and newer queries will block until there
# is space on the queue.
# If a value of 0 is used, the queue size will be unbounded, which can lead to resource exhaustion under
# heavy query load.
queue_size: 10000
# the consistency level used to write audit events
write_consistency: QUORUM
# if enabled, system tables that may contain sensitive information (system.hints,
# system.batchlog, system.paxos) are encrypted with the encryption settings below.
# When enabling system table encryption on a node with existing data, run
# `nodetool upgradesstables -a` on the listed tables to encrypt existing data
#
# When tracing is enabled, sensitive info will be written into the tables in the
# system_traces keyspace. Those tables should be configured to encrypt their data
# on disk by using an encrypting compressor.
system_info_encryption:
enabled: false
cipher_algorithm: AES
secret_key_strength: 128
chunk_length_kb: 64
# the name of the keys file that will be created to encrypt system tables.
# This file will be created at /system/
key_name: system_table_keytab
# Retries setting when hive inserts data to C* table. insert_max_retries is max number of retries
# insert_retry_sleep_period is the period of time in milliseconds between retries
hive_options:
insert_max_retries: 6
insert_retry_sleep_period: 50
What is the value of initial_spark_worker_resources?3. /etc/dse/cassandra/cassandra.yaml
# Cassandra storage config YAML
# NOTE:
# See http://wiki.apache.org/cassandra/StorageConfiguration for
# full explanations of configuration directives
# /NOTE
# The name of the cluster. This is mainly used to prevent machines in
# one logical cluster from joining another.
cluster_name: "Cluster 1"
# This defines the number of tokens randomly assigned to this node on the ring
# The more tokens, relative to other nodes, the larger the proportion of data
# that this node will store. You probably want all nodes to have the same number
# of tokens assuming they have equal hardware capability.
#
# If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility,
# and will use the initial_token as described below.
#
# Specifying initial_token will override this setting on the node's initial start,
# on subsequent starts, this setting will apply even if initial token is set.
#
# If you already have a cluster with 1 token per node, and wish to migrate to
# multiple tokens per node, see http://wiki.apache.org/cassandra/Operations
# num_tokens:
# initial_token allows you to specify tokens manually. While you can use # it with
# vnodes (num_tokens > 1, above) -- in which case you should provide a
# comma-separated list -- it's primarily used when adding nodes # to legacy clusters
# that do not have vnodes enabled.
initial_token: -9223372036854775808
# May either be "true" or "false" to enable globally, or contain a list
# of data centers to enable per-datacenter.
# hinted_handoff_enabled: DC1,DC2
# See http://wiki.apache.org/cassandra/HintedHandoff
hinted_handoff_enabled: true
# this defines the maximum amount of time a dead host will have hints
# generated. After it has been dead this long, new hints for it will not be
# created until it has been seen alive and gone down again.
max_hint_window_in_ms: 10800000 # 3 hours
# Maximum throttle in KBs per second, per delivery thread. This will be
# reduced proportionally to the number of nodes in the cluster. (If there
# are two nodes in the cluster, each delivery thread will use the maximum
# rate; if there are three, each will throttle to half of the maximum,
# since we expect two nodes to be delivering hints simultaneously.)
hinted_handoff_throttle_in_kb: 1024
# Number of threads with which to deliver hints;
# Consider increasing this number when you have multi-dc deployments, since
# cross-dc handoff tends to be slower
max_hints_delivery_threads: 2
# Maximum throttle in KBs per second, total. This will be
# reduced proportionally to the number of nodes in the cluster.
batchlog_replay_throttle_in_kb: 1024
# Authentication backend, implementing IAuthenticator; used to identify users
# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator,
# PasswordAuthenticator}.
# DSE also provides Kerberos and Ldap authenticators for external authentication.
#
# - AllowAllAuthenticator performs no checks - set it to disable authentication.
# - PasswordAuthenticator relies on username/password pairs to authenticate
# users. It keeps usernames and hashed passwords in system_auth.credentials table.
# Please increase system_auth keyspace replication factor if you use this authenticator.
# - com.datastax.bdp.cassandra.auth.KerberosAuthenticator For external authentication
# through Kerberos. Additional configuration is required in dse.yaml
# - com.datastax.bdp.cassandra.auth.LdapAuthenticator For external authentication
# through Ldap. Additional configuration is required in dse.yaml
authenticator: AllowAllAuthenticator
#authenticator: com.datastax.bdp.cassandra.auth.KerberosAuthenticator
#authenticator: com.datastax.bdp.cassandra.auth.LdapAuthenticator
# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions
# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer,
# CassandraAuthorizer}.
#
# - AllowAllAuthorizer allows any action to any user - set it to disable authorization.
# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please
# increase system_auth keyspace replication factor if you use this authorizer.
authorizer: AllowAllAuthorizer
# Validity period for permissions cache (fetching permissions can be an
# expensive operation depending on the authorizer, CassandraAuthorizer is
# one example). Defaults to 2000, set to 0 to disable.
# Will be disabled automatically for AllowAllAuthorizer.
permissions_validity_in_ms: 2000
# Refresh interval for permissions cache (if enabled).
# After this interval, cache entries become eligible for refresh. Upon next
# access, an async reload is scheduled and the old value returned until it
# completes. If permissions_validity_in_ms is non-zero, then this must be
# also.
# Defaults to the same value as permissions_validity_in_ms.
# permissions_update_interval_in_ms: 1000
# The partitioner is responsible for distributing groups of rows (by
# partition key) across nodes in the cluster. You should leave this
# alone for new clusters. The partitioner can NOT be changed without
# reloading all data, so when upgrading you should set this to the
# same partitioner you were already using.
#
# Besides Murmur3Partitioner, partitioners included for backwards
# compatibility include RandomPartitioner, ByteOrderedPartitioner, and
# OrderPreservingPartitioner.
#
partitioner: org.apache.cassandra.dht.Murmur3Partitioner
# Directories where Cassandra should store data on disk. Cassandra
# will spread data evenly across them, subject to the granularity of
# the configured compaction strategy.
data_file_directories:
- /mnt/ephemeral/cassandra/data
# commit log
commitlog_directory: "/mnt/ephemeral/cassandra/commitlog"
# policy for data disk failures:
# stop_paranoid: shut down gossip and Thrift even for single-sstable errors.
# stop: shut down gossip and Thrift, leaving the node effectively dead, but
# can still be inspected via JMX.
# best_effort: stop using the failed disk and respond to requests based on
# remaining available sstables. This means you WILL see obsolete
# data at CL.ONE!
# ignore: ignore fatal errors and let requests fail, as in pre-1.2 Cassandra
disk_failure_policy: stop
# policy for commit disk failures:
# stop: shut down gossip and Thrift, leaving the node effectively dead, but
# can still be inspected via JMX.
# stop_commit: shutdown the commit log, letting writes collect but
# continuing to service reads, as in pre-2.0.5 Cassandra
# ignore: ignore fatal errors and let the batches fail
commit_failure_policy: stop
# Maximum size of the key cache in memory.
#
# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the
# minimum, sometimes more. The key cache is fairly tiny for the amount of
# time it saves, so it's worthwhile to use it at large numbers.
# The row cache saves even more time, but must contain the entire row,
# so it is extremely space-intensive. It's best to only use the
# row cache if you have hot rows or static rows.
#
# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup.
#
# Default value is empty to make it "auto" (min(5% of Heap (in MB), 100MB)). Set to 0 to disable key cache.
key_cache_size_in_mb:
# Duration in seconds after which Cassandra should
# save the key cache. Caches are saved to saved_caches_directory as
# specified in this configuration file.
#
# Saved caches greatly improve cold-start speeds, and is relatively cheap in
# terms of I/O for the key cache. Row cache saving is much more expensive and
# has limited use.
#
# Default is 14400 or 4 hours.
key_cache_save_period: 14400
# Number of keys from the key cache to save
# Disabled by default, meaning all keys are going to be saved
# key_cache_keys_to_save: 100
# Maximum size of the row cache in memory.
# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup.
#
# Default value is 0, to disable row caching.
row_cache_size_in_mb: 0
# Duration in seconds after which Cassandra should
# safe the row cache. Caches are saved to saved_caches_directory as specified
# in this configuration file.
#
# Saved caches greatly improve cold-start speeds, and is relatively cheap in
# terms of I/O for the key cache. Row cache saving is much more expensive and
# has limited use.
#
# Default is 0 to disable saving the row cache.
row_cache_save_period: 0
# Number of keys from the row cache to save
# Disabled by default, meaning all keys are going to be saved
# row_cache_keys_to_save: 100
# The off-heap memory allocator. Affects storage engine metadata as
# well as caches. Experiments show that JEMAlloc saves some memory
# than the native GCC allocator (i.e., JEMalloc is more
# fragmentation-resistant).
#
# Supported values are: NativeAllocator, JEMallocAllocator
#
# If you intend to use JEMallocAllocator you have to install JEMalloc as library and
# modify cassandra-env.sh as directed in the file.
#
# Defaults to NativeAllocator
# memory_allocator: NativeAllocator
# saved caches
saved_caches_directory: "/mnt/ephemeral/cassandra/saved_caches"
# commitlog_sync may be either "periodic" or "batch."
# When in batch mode, Cassandra won't ack writes until the commit log
# has been fsynced to disk. It will wait up to
# commitlog_sync_batch_window_in_ms milliseconds for other writes, before
# performing the sync.
#
# commitlog_sync: batch
# commitlog_sync_batch_window_in_ms: 50
#
# the other option is "periodic" where writes may be acked immediately
# and the CommitLog is simply synced every commitlog_sync_period_in_ms
# milliseconds. By default this allows 1024*(CPU cores) pending
# entries on the commitlog queue. If you are writing very large blobs,
# you should reduce that; 16*cores works reasonably well for 1MB blobs.
# It should be at least as large as the concurrent_writes setting.
commitlog_sync: periodic
commitlog_sync_period_in_ms: 10000
# commitlog_periodic_queue_size:
# The size of the individual commitlog file segments. A commitlog
# segment may be archived, deleted, or recycled once all the data
# in it (potentially from each columnfamily in the system) has been
# flushed to sstables.
#
# The default size is 32, which is almost always fine, but if you are
# archiving commitlog segments (see commitlog_archiving.properties),
# then you probably want a finer granularity of archiving; 8 or 16 MB
# is reasonable.
commitlog_segment_size_in_mb: 32
# any class that implements the SeedProvider interface and has a
# constructor that takes a Map of parameters will do.
seed_provider:
# Addresses of hosts that are deemed contact points.
# Cassandra nodes use this list of hosts to find each other and learn
# the topology of the ring. You must change this if you are running
# multiple nodes!
- class_name: org.apache.cassandra.locator.SimpleSeedProvider
parameters:
# seeds is actually a comma-delimited list of addresses.
# Ex: ",,"
- seeds: "172.31.5.11"
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
# disk. "concurrent_reads" should be set to (16 * number_of_drives) in
# order to allow the operations to enqueue low enough in the stack
# that the OS and drives can reorder them.
#
# On the other hand, since writes are almost never IO bound, the ideal
# number of "concurrent_writes" is dependent on the number of cores in
# your system; (8 * number_of_cores) is a good rule of thumb.
concurrent_reads: 32
concurrent_writes: 32
# Total memory to use for sstable-reading buffers. Defaults to
# the smaller of 1/4 of heap or 512MB.
# file_cache_size_in_mb: 512
# Total memory to use for memtables. Cassandra will flush the largest
# memtable when this much memory is used.
# If omitted, Cassandra will set it to 1/4 of the heap.
# memtable_total_space_in_mb: 2048
# Total space to use for commitlogs. Since commitlog segments are
# mmapped, and hence use up address space, the default size is 32
# on 32-bit JVMs, and 1024 on 64-bit JVMs.
#
# If space gets above this value (it will round up to the next nearest
# segment multiple), Cassandra will flush every dirty CF in the oldest
# segment and remove it. So a small total commitlog space will tend
# to cause more flush activity on less-active columnfamilies.
# commitlog_total_space_in_mb: 4096
# This sets the amount of memtable flush writer threads. These will
# be blocked by disk io, and each one will hold a memtable in memory
# while blocked. If you have a large heap and many data directories,
# you can increase this value for better flush performance.
# By default this will be set to the amount of data directories defined.
#memtable_flush_writers: 1
# the number of full memtables to allow pending flush, that is,
# waiting for a writer thread. At a minimum, this should be set to
# the maximum number of secondary indexes created on a single CF.
memtable_flush_queue_size: 4
# Whether to, when doing sequential writing, fsync() at intervals in
# order to force the operating system to flush the dirty
# buffers. Enable this to avoid sudden dirty buffer flushing from
# impacting read latencies. Almost always a good idea on SSDs; not
# necessarily on platters.
trickle_fsync: false
trickle_fsync_interval_in_kb: 10240
# TCP port, for commands and data
storage_port: 7000
# SSL port, for encrypted communication. Unused unless enabled in
# encryption_options
ssl_storage_port: 7001
# Address to bind to and tell other Cassandra nodes to connect to. You
# _must_ change this if you want multiple nodes to be able to
# communicate!
#
# Leaving it blank leaves it up to InetAddress.getLocalHost(). This
# will always do the Right Thing _if_ the node is properly configured
# (hostname, name resolution, etc), and the Right Thing is to use the
# address associated with the hostname (it might not be).
#
# Setting this to 0.0.0.0 is always wrong.
listen_address: 172.31.5.11
# Address to broadcast to other Cassandra nodes
# Leaving this blank will set it to the same value as listen_address
# broadcast_address: 1.2.3.4
# Internode authentication backend, implementing IInternodeAuthenticator;
# used to allow/disallow connections from peer nodes.
# internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator
# Whether to start the native transport server.
# Please note that the address on which the native transport is bound is the
# same as the rpc_address. The port however is different and specified below.
start_native_transport: true
# port for the CQL native transport to listen for clients on
native_transport_port: 9042
# The maximum threads for handling requests when the native transport is used.
# This is similar to rpc_max_threads though the default differs slightly (and
# there is no native_transport_min_threads, idle threads will always be stopped
# after 30 seconds).
# native_transport_max_threads: 128
#
# The maximum size of allowed frame. Frame (requests) larger than this will
# be rejected as invalid. The default is 256MB.
# native_transport_max_frame_size_in_mb: 256
# Whether to start the thrift rpc server.
start_rpc: true
# The address to bind the Thrift RPC service and native transport
# server -- clients connect here.
#
# Leaving this blank has the same effect it does for ListenAddress,
# (i.e. it will be based on the configured hostname of the node).
#
# Note that unlike ListenAddress above, it is allowed to specify 0.0.0.0
# here if you want to listen on all interfaces, but that will break clients
# that rely on node auto-discovery.
rpc_address: 172.31.5.11
# port for Thrift to listen for clients on
rpc_port: 9160
# enable or disable keepalive on rpc/native connections
rpc_keepalive: true
# Cassandra provides two out-of-the-box options for the RPC Server:
#
# sync -> One thread per thrift connection. For a very large number of clients, memory
# will be your limiting factor. On a 64 bit JVM, 180KB is the minimum stack size
# per thread, and that will correspond to your use of virtual memory (but physical memory
# may be limited depending on use of stack space).
#
# hsha -> Stands for "half synchronous, half asynchronous." All thrift clients are handled
# asynchronously using a small number of threads that does not vary with the amount
# of thrift clients (and thus scales well to many clients). The rpc requests are still
# synchronous (one thread per active request). If hsha is selected then it is essential
# that rpc_max_threads is changed from the default value of unlimited.
#
# The default is sync because on Windows hsha is about 30% slower. On Linux,
# sync/hsha performance is about the same, with hsha of course using less memory.
#
# Alternatively, can provide your own RPC server by providing the fully-qualified class name
# of an o.a.c.t.TServerFactory that can create an instance of it.
rpc_server_type: sync
# Uncomment rpc_min|max_thread to set request pool size limits.
#
# Regardless of your choice of RPC server (see above), the number of maximum requests in the
# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync
# RPC server, it also dictates the number of clients that can be connected at all).
#
# The default is unlimited and thus provides no protection against clients overwhelming the server. You are
# encouraged to set a maximum that makes sense for you in production, but do keep in mind that
# rpc_max_threads represents the maximum number of client requests this server may execute concurrently.
#
# rpc_min_threads: 16
# rpc_max_threads: 2048
# uncomment to set socket buffer sizes on rpc connections
# rpc_send_buff_size_in_bytes:
# rpc_recv_buff_size_in_bytes:
# Uncomment to set socket buffer size for internode communication
# Note that when setting this, the buffer size is limited by net.core.wmem_max
# and when not setting it it is defined by net.ipv4.tcp_wmem
# See:
# /proc/sys/net/core/wmem_max
# /proc/sys/net/core/rmem_max
# /proc/sys/net/ipv4/tcp_wmem
# /proc/sys/net/ipv4/tcp_wmem
# and: man tcp
# internode_send_buff_size_in_bytes:
# internode_recv_buff_size_in_bytes:
# Frame size for thrift (maximum message length).
thrift_framed_transport_size_in_mb: 15
# Set to true to have Cassandra create a hard link to each sstable
# flushed or streamed locally in a backups/ subdirectory of the
# keyspace data. Removing these links is the operator's
# responsibility.
incremental_backups: false
# Whether or not to take a snapshot before each compaction. Be
# careful using this option, since Cassandra won't clean up the
# snapshots for you. Mostly useful if you're paranoid when there
# is a data format change.
snapshot_before_compaction: false
# Whether or not a snapshot is taken of the data before keyspace truncation
# or dropping of column families. The STRONGLY advised default of true
# should be used to provide data safety. If you set this flag to false, you will
# lose data on truncation or drop.
auto_snapshot: true
# When executing a scan, within or across a partition, we need to keep the
# tombstones seen in memory so we can return them to the coordinator, which
# will use them to make sure other replicas also know about the deleted rows.
# With workloads that generate a lot of tombstones, this can cause performance
# problems and even exaust the server heap.
# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
# Adjust the thresholds here if you understand the dangers and want to
# scan more tombstones anyway. These thresholds may also be adjusted at runtime
# using the StorageService mbean.
tombstone_warn_threshold: 1000
tombstone_failure_threshold: 100000
# Granularity of the collation index of rows within a partition.
# Increase if your rows are large, or if you have a very large
# number of rows per partition. The competing goals are these:
# 1) a smaller granularity means more index entries are generated
# and looking up rows withing the partition by collation column
# is faster
# 2) but, Cassandra will keep the collation index in memory for hot
# rows (as part of the key cache), so a larger granularity means
# you can cache more hot rows
column_index_size_in_kb: 64
# Log WARN on any batch size exceeding this value. 64kb per batch by default.
# Caution should be taken on increasing the size of this threshold as it can lead to node instability.
batch_size_warn_threshold_in_kb: 64
# Size limit for rows being compacted in memory. Larger rows will spill
# over to disk and use a slower two-pass compaction process. A message
# will be logged specifying the row key.
in_memory_compaction_limit_in_mb: 64
# Number of simultaneous compactions to allow, NOT including
# validation "compactions" for anti-entropy repair. Simultaneous
# compactions can help preserve read performance in a mixed read/write
# workload, by mitigating the tendency of small sstables to accumulate
# during a single long running compactions. The default is usually
# fine and if you experience problems with compaction running too
# slowly or too fast, you should look at
# compaction_throughput_mb_per_sec first.
#
# concurrent_compactors defaults to the number of cores.
# Uncomment to make compaction mono-threaded, the pre-0.8 default.
#concurrent_compactors: 1
# Multi-threaded compaction. When enabled, each compaction will use
# up to one thread per core, plus one thread per sstable being merged.
# This is usually only useful for SSD-based hardware: otherwise,
# your concern is usually to get compaction to do LESS i/o (see:
# compaction_throughput_mb_per_sec), not more.
multithreaded_compaction: false
# Throttles compaction to the given total throughput across the entire
# system. The faster you insert data, the faster you need to compact in
# order to keep the sstable count down, but in general, setting this to
# 16 to 32 times the rate you are inserting data is more than sufficient.
# Setting this to 0 disables throttling. Note that this account for all types
# of compaction, including validation compaction.
compaction_throughput_mb_per_sec: 16
# Track cached row keys during compaction, and re-cache their new
# positions in the compacted sstable. Disable if you use really large
# key caches.
compaction_preheat_key_cache: true
# Throttles all outbound streaming file transfers on this node to the
# given total throughput in Mbps. This is necessary because Cassandra does
# mostly sequential IO when streaming data during bootstrap or repair, which
# can lead to saturating the network connection and degrading rpc performance.
# When unset, the default is 200 Mbps or 25 MB/s.
stream_throughput_outbound_megabits_per_sec: 200
# Throttles all streaming file transfer between the datacenters,
# this setting allows users to throttle inter dc stream throughput in addition
# to throttling all network stream traffic as configured with
# stream_throughput_outbound_megabits_per_sec
inter_dc_stream_throughput_outbound_megabits_per_sec: 200
# How long the coordinator should wait for read operations to complete
read_request_timeout_in_ms: 5000
# How long the coordinator should wait for seq or index scans to complete
range_request_timeout_in_ms: 10000
# How long the coordinator should wait for writes to complete
write_request_timeout_in_ms: 2000
# How long a coordinator should continue to retry a CAS operation
# that contends with other proposals for the same row
cas_contention_timeout_in_ms: 1000
# How long the coordinator should wait for truncates to complete
# (This can be much longer, because unless auto_snapshot is disabled
# we need to flush first so we can snapshot before removing the data.)
truncate_request_timeout_in_ms: 60000
# The default timeout for other, miscellaneous operations
request_timeout_in_ms: 10000
# Enable operation timeout information exchange between nodes to accurately
# measure request timeouts. If disabled, replicas will assume that requests
# were forwarded to them instantly by the coordinator, which means that
# under overload conditions we will waste that much extra time processing
# already-timed-out requests.
#
# Warning: before enabling this property make sure to ntp is installed
# and the times are synchronized between the nodes.
cross_node_timeout: false
# Enable socket timeout for streaming operation.
# When a timeout occurs during streaming, streaming is retried from the start
# of the current file. This _can_ involve re-streaming an important amount of
# data, so you should avoid setting the value too low.
# Default value is 0, which never timeout streams.
# streaming_socket_timeout_in_ms: 0
# phi value that must be reached for a host to be marked down.
# most users should never need to adjust this.
phi_convict_threshold: 12
# endpoint_snitch -- Set this to a class that implements
# IEndpointSnitch. The snitch has two functions:
# - it teaches Cassandra enough about your network topology to route
# requests efficiently
# - it allows Cassandra to spread replicas around your cluster to avoid
# correlated failures. It does this by grouping machines into
# "datacenters" and "racks." Cassandra will do its best not to have
# more than one replica on the same "rack" (which may not actually
# be a physical location)
#
# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER,
# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS
# ARE PLACED.
#
# Out of the box, Cassandra provides
# - SimpleSnitch:
# Treats Strategy order as proximity. This can improve cache
# locality when disabling read repair. Only appropriate for
# single-datacenter deployments.
# - GossipingPropertyFileSnitch
# This should be your go-to snitch for production use. The rack
# and datacenter for the local node are defined in
# cassandra-rackdc.properties and propagated to other nodes via
# gossip. If cassandra-topology.properties exists, it is used as a
# fallback, allowing migration from the PropertyFileSnitch.
# - PropertyFileSnitch:
# Proximity is determined by rack and data center, which are
# explicitly configured in cassandra-topology.properties.
# - Ec2Snitch:
# Appropriate for EC2 deployments in a single Region. Loads Region
# and Availability Zone information from the EC2 API. The Region is
# treated as the datacenter, and the Availability Zone as the rack.
# Only private IPs are used, so this will not work across multiple
# Regions.
# - Ec2MultiRegionSnitch:
# Uses public IPs as broadcast_address to allow cross-region
# connectivity. (Thus, you should set seed addresses to the public
# IP as well.) You will need to open the storage_port or
# ssl_storage_port on the public IP firewall. (For intra-Region
# traffic, Cassandra will switch to the private IP after
# establishing a connection.)
# - RackInferringSnitch:
# Proximity is determined by rack and data center, which are
# assumed to correspond to the 3rd and 2nd octet of each node's IP
# address, respectively. Unless this happens to match your
# deployment conventions, this is best used as an example of
# writing a custom Snitch class and is provided in that spirit.
# DataStax Enterprise provides
# - com.datastax.bdp.snitch.DseSimpleSnitch:
# Proximity is determined by DSE workload, which places Cassandra,
# Analytics, and Solr nodes into their seperate datacenters.
#
# You can use a custom Snitch by setting this to the full class name
# of the snitch, which will be assumed to be on your classpath.
endpoint_snitch: "com.datastax.bdp.snitch.DseSimpleSnitch"
# controls how often to perform the more expensive part of host score
# calculation
dynamic_snitch_update_interval_in_ms: 100
# controls how often to reset all host scores, allowing a bad host to
# possibly recover
dynamic_snitch_reset_interval_in_ms: 600000
# if set greater than zero and read_repair_chance is < 1.0, this will allow
# 'pinning' of replicas to hosts in order to increase cache capacity.
# The badness threshold will control how much worse the pinned host has to be
# before the dynamic snitch will prefer other replicas over it. This is
# expressed as a double which represents a percentage. Thus, a value of
# 0.2 means Cassandra would continue to prefer the static snitch values
# until the pinned host was 20% worse than the fastest.
dynamic_snitch_badness_threshold: 0.1
# request_scheduler -- Set this to a class that implements
# RequestScheduler, which will schedule incoming client requests
# according to the specific policy. This is useful for multi-tenancy
# with a single Cassandra cluster.
# NOTE: This is specifically for requests from the client and does
# not affect inter node communication.
# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place
# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of
# client requests to a node with a separate queue for each
# request_scheduler_id. The scheduler is further customized by
# request_scheduler_options as described below.
request_scheduler: org.apache.cassandra.scheduler.NoScheduler
# Scheduler Options vary based on the type of scheduler
# NoScheduler - Has no options
# RoundRobin
# - throttle_limit -- The throttle_limit is the number of in-flight
# requests per client. Requests beyond
# that limit are queued up until
# running requests can complete.
# The value of 80 here is twice the number of
# concurrent_reads + concurrent_writes.
# - default_weight -- default_weight is optional and allows for
# overriding the default which is 1.
# - weights -- Weights are optional and will default to 1 or the
# overridden default_weight. The weight translates into how
# many requests are handled during each turn of the
# RoundRobin, based on the scheduler id.
#
# request_scheduler_options:
# throttle_limit: 80
# default_weight: 5
# weights:
# Keyspace1: 1
# Keyspace2: 5
# request_scheduler_id -- An identifier based on which to perform
# the request scheduling. Currently the only valid option is keyspace.
# request_scheduler_id: keyspace
# Enable or disable inter-node encryption
# Default settings are TLS v1, RSA 1024-bit keys (it is imperative that
# users generate their own keys) TLS_RSA_WITH_AES_128_CBC_SHA as the cipher
# suite for authentication, key exchange and encryption of the actual data transfers.
# Use the DHE/ECDHE ciphers if running in FIPS 140 compliant mode.
# NOTE: No custom encryption options are enabled at the moment
# The available internode options are : all, none, dc, rack
#
# If set to dc cassandra will encrypt the traffic between the DCs
# If set to rack cassandra will encrypt the traffic between the racks
#
# The passwords used in these options must match the passwords used when generating
# the keystore and truststore. For instructions on generating these files, see:
# http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
#
server_encryption_options:
internode_encryption: none
keystore: resources/dse/conf/.keystore
keystore_password: cassandra
truststore: resources/dse/conf/.truststore
truststore_password: cassandra
# More advanced defaults below:
# protocol: TLS
# algorithm: SunX509
# store_type: JKS
# cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]
# require_client_auth: false
# enable or disable client/server encryption.
client_encryption_options:
enabled: false
keystore: resources/dse/conf/.keystore
keystore_password: cassandra
# require_client_auth: false
# Set trustore and truststore_password if require_client_auth is true
# truststore: resources/dse/conf/.truststore
# truststore_password: cassandra
# More advanced defaults below:
# protocol: TLS
# algorithm: SunX509
# store_type: JKS
# cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]
# internode_compression controls whether traffic between nodes is
# compressed.
# can be: all - all traffic is compressed
# dc - traffic between different datacenters is compressed
# none - nothing is compressed.
internode_compression: all
# Enable or disable tcp_nodelay for inter-dc communication.
# Disabling it will result in larger (but fewer) network packets being sent,
# reducing overhead from the TCP protocol itself, at the cost of increasing
# latency if you block for cross-datacenter responses.
inter_dc_tcp_nodelay: false
# Enable or disable kernel page cache preheating from contents of the key cache after compaction.
# When enabled it would preheat only first "page" (4KB) of each row to optimize
# for sequential access. Note: This could be harmful for fat rows, see CASSANDRA-4937
# for further details on that topic.
preheat_kernel_page_cache: false
What are the values of cluster_name, num_tokens, and initial_token?4. /etc/dse/cassandra/cassandra-env.sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
calculate_heap_sizes()
{
case "`uname`" in
Linux)
system_memory_in_mb=`free -m | awk '/:/ {print $2;exit}'`
system_cpu_cores=`egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo`
;;
FreeBSD)
system_memory_in_bytes=`sysctl hw.physmem | awk '{print $2}'`
system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
;;
SunOS)
system_memory_in_mb=`prtconf | awk '/Memory size:/ {print $3}'`
system_cpu_cores=`psrinfo | wc -l`
;;
Darwin)
system_memory_in_bytes=`sysctl hw.memsize | awk '{print $2}'`
system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
;;
*)
# assume reasonable defaults for e.g. a modern desktop or
# cheap server
system_memory_in_mb="2048"
system_cpu_cores="2"
;;
esac
# some systems like the raspberry pi don't report cores, use at least 1
if [ "$system_cpu_cores" -lt "1" ]
then
system_cpu_cores="1"
fi
# set max heap size based on the following
# max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB))
# calculate 1/2 ram and cap to 1024MB
# calculate 1/4 ram and cap to 8192MB
# pick the max
half_system_memory_in_mb=`expr $system_memory_in_mb / 2`
quarter_system_memory_in_mb=`expr $half_system_memory_in_mb / 2`
if [ "$half_system_memory_in_mb" -gt "1024" ]
then
half_system_memory_in_mb="1024"
fi
if [ "$quarter_system_memory_in_mb" -gt "8192" ]
then
quarter_system_memory_in_mb="8192"
fi
if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ]
then
max_heap_size_in_mb="$half_system_memory_in_mb"
else
max_heap_size_in_mb="$quarter_system_memory_in_mb"
fi
MAX_HEAP_SIZE="${max_heap_size_in_mb}M"
# These env variables may be useful to determine proper amount of resources
# assigned to DSE subprocesses, such as Spark components
SYSTEM_CPU_CORES="$system_cpu_cores"
export SYSTEM_CPU_CORES
SYSTEM_MEMORY_IN_MB="$system_memory_in_mb"
export SYSTEM_MEMORY_IN_MB
CASSANDRA_MEMORY_IN_MB="1024"
export CASSANDRA_MEMORY_IN_MB
# Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4 * heap size)
max_sensible_yg_per_core_in_mb="100"
max_sensible_yg_in_mb=`expr $max_sensible_yg_per_core_in_mb "*" $system_cpu_cores`
desired_yg_in_mb=`expr $max_heap_size_in_mb / 4`
if [ "$desired_yg_in_mb" -gt "$max_sensible_yg_in_mb" ]
then
HEAP_NEWSIZE="${max_sensible_yg_in_mb}M"
else
HEAP_NEWSIZE="${desired_yg_in_mb}M"
fi
}
# Determine the sort of JVM we'll be running on.
java_ver_output=`"${JAVA:-java}" -version 2>&1`
jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}'`
JVM_VERSION=${jvmver%_*}
JVM_PATCH_VERSION=${jvmver#*_}
if [ "$JVM_VERSION" \< "1.7" ] ; then
echo "Cassandra 2.0 and later require Java 7 or later."
exit 1;
fi
jvm=`echo "$java_ver_output" | grep -A 1 'java version' | awk 'NR==2 {print $1}'`
case "$jvm" in
OpenJDK)
JVM_VENDOR=OpenJDK
# this will be "64-Bit" or "32-Bit"
JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $2}'`
;;
"Java(TM)")
JVM_VENDOR=Oracle
# this will be "64-Bit" or "32-Bit"
JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $3}'`
;;
*)
# Help fill in other JVM values
JVM_VENDOR=other
JVM_ARCH=unknown
;;
esac
# Override these to set the amount of memory to allocate to the JVM at
# start-up. For production use you may wish to adjust this for your
# environment. MAX_HEAP_SIZE is the total amount of memory dedicated
# to the Java heap; HEAP_NEWSIZE refers to the size of the young
# generation. Both MAX_HEAP_SIZE and HEAP_NEWSIZE should be either set
# or not (if you set one, set the other).
#
# The main trade-off for the young generation is that the larger it
# is, the longer GC pause times will be. The shorter it is, the more
# expensive GC will be (usually).
#
# The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent pause
# times. If in doubt, and if you do not particularly want to tweak, go with
# 100 MB per physical CPU core.
MAX_HEAP_SIZE="1G"
HEAP_NEWSIZE="200M"
# Set this to control the amount of arenas per-thread in glibc
#export MALLOC_ARENA_MAX=4
if [ "x$MAX_HEAP_SIZE" = "x" ] && [ "x$HEAP_NEWSIZE" = "x" ]; then
calculate_heap_sizes
else
if [ "x$MAX_HEAP_SIZE" = "x" ] || [ "x$HEAP_NEWSIZE" = "x" ]; then
echo "please set or unset MAX_HEAP_SIZE and HEAP_NEWSIZE in pairs (see cassandra-env.sh)"
exit 1
fi
fi
if [ "x$MALLOC_ARENA_MAX" = "x" ]
then
export MALLOC_ARENA_MAX=4
fi
# Specifies the default port over which Cassandra will be available for
# JMX connections.
JMX_PORT="7199"
# Here we create the arguments that will get passed to the jvm when
# starting cassandra.
# enable assertions. disabling this in production will give a modest
# performance benefit (around 5%).
JVM_OPTS="$JVM_OPTS -ea"
# Special-case path variables for Windows.
case "`uname`" in
CYGWIN*)
CASSANDRA_HOME=`cygpath -p -w "$CASSANDRA_HOME"`
;;
esac
# add the DSE loader
JVM_OPTS="$JVM_OPTS $DSE_OPTS $DSE_CREDENTIALS"
# add the jamm javaagent
JVM_OPTS="$JVM_OPTS -javaagent:$CASSANDRA_HOME/lib/jamm-0.2.5.jar"
# some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
JVM_OPTS="$JVM_OPTS -XX:+CMSClassUnloadingEnabled"
# enable thread priorities, primarily so we can give periodic tasks
# a lower priority to avoid interfering with client workload
JVM_OPTS="$JVM_OPTS -XX:+UseThreadPriorities"
# allows lowering thread priority without being root. see
# http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workaround.html
JVM_OPTS="$JVM_OPTS -XX:ThreadPriorityPolicy=42"
# min and max heap sizes should be set to the same value to avoid
# stop-the-world GC pauses during resize, and so that we can lock the
# heap in memory on startup to prevent any of it from being swapped
# out.
JVM_OPTS="$JVM_OPTS -Xms${MAX_HEAP_SIZE}"
JVM_OPTS="$JVM_OPTS -Xmx${MAX_HEAP_SIZE}"
JVM_OPTS="$JVM_OPTS -Xmn${HEAP_NEWSIZE}"
JVM_OPTS="$JVM_OPTS -XX:+HeapDumpOnOutOfMemoryError"
# set jvm HeapDumpPath with CASSANDRA_HEAPDUMP_DIR
if [ "x$CASSANDRA_HEAPDUMP_DIR" != "x" ]; then
JVM_OPTS="$JVM_OPTS -XX:HeapDumpPath=$CASSANDRA_HEAPDUMP_DIR/cassandra-`date +%s`-pid$$.hprof"
fi
startswith() { [ "${1#$2}" != "$1" ]; }
# Per-thread stack size.
JVM_OPTS="$JVM_OPTS -Xss256k"
# Larger interned string table, for gossip's benefit (CASSANDRA-6410)
JVM_OPTS="$JVM_OPTS -XX:StringTableSize=1000003"
# GC tuning options
JVM_OPTS="$JVM_OPTS -XX:+UseParNewGC"
JVM_OPTS="$JVM_OPTS -XX:+UseConcMarkSweepGC"
JVM_OPTS="$JVM_OPTS -XX:+CMSParallelRemarkEnabled"
JVM_OPTS="$JVM_OPTS -XX:SurvivorRatio=8"
JVM_OPTS="$JVM_OPTS -XX:MaxTenuringThreshold=1"
JVM_OPTS="$JVM_OPTS -XX:CMSInitiatingOccupancyFraction=75"
JVM_OPTS="$JVM_OPTS -XX:+UseCMSInitiatingOccupancyOnly"
JVM_OPTS="$JVM_OPTS -XX:+UseTLAB"
# note: bash evals '1.7.x' as > '1.7' so this is really a >= 1.7 jvm check
if { [ "$JVM_VERSION" \> "1.7" ] && [ "$JVM_VERSION" \< "1.8.0" ] && [ "$JVM_PATCH_VERSION" -ge "60" ]; } || [ "$JVM_VERSION" \> "1.8" ] ; then
JVM_OPTS="$JVM_OPTS -XX:+CMSParallelInitialMarkEnabled -XX:+CMSEdenChunksRecordAlways"
fi
if [ "$JVM_ARCH" = "64-Bit" ] ; then
JVM_OPTS="$JVM_OPTS -XX:+UseCondCardMark"
fi
# GC logging options -- uncomment to enable
# JVM_OPTS="$JVM_OPTS -XX:+PrintGCDetails"
# JVM_OPTS="$JVM_OPTS -XX:+PrintGCDateStamps"
# JVM_OPTS="$JVM_OPTS -XX:+PrintHeapAtGC"
# JVM_OPTS="$JVM_OPTS -XX:+PrintTenuringDistribution"
# JVM_OPTS="$JVM_OPTS -XX:+PrintGCApplicationStoppedTime"
# JVM_OPTS="$JVM_OPTS -XX:+PrintPromotionFailure"
# JVM_OPTS="$JVM_OPTS -XX:PrintFLSStatistics=1"
# JVM_OPTS="$JVM_OPTS -Xloggc:/var/log/cassandra/gc-`date +%s`.log"
# If you are using JDK 6u34 7u2 or later you can enable GC log rotation
# don't stick the date in the log name if rotation is on.
# JVM_OPTS="$JVM_OPTS -Xloggc:/var/log/cassandra/gc.log"
# JVM_OPTS="$JVM_OPTS -XX:+UseGCLogFileRotation"
# JVM_OPTS="$JVM_OPTS -XX:NumberOfGCLogFiles=10"
# JVM_OPTS="$JVM_OPTS -XX:GCLogFileSize=10M"
# Configure the following for JEMallocAllocator and if jemalloc is not available in the system
# library path (Example: /usr/local/lib/). Usually "make install" will do the right thing.
# export LD_LIBRARY_PATH=/lib/
# JVM_OPTS="$JVM_OPTS -Djava.library.path=/lib/"
# uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414
# JVM_OPTS="$JVM_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1414"
# Prefer binding to IPv4 network intefaces (when net.ipv6.bindv6only=1). See
# http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342561 (short version:
# comment out this entry to enable IPv6 support).
JVM_OPTS="$JVM_OPTS -Djava.net.preferIPv4Stack=true"
# jmx: metrics and administration interface
#
# add this if you're having trouble connecting:
# JVM_OPTS="$JVM_OPTS -Djava.rmi.server.hostname="
#
# see
# https://blogs.oracle.com/jmxetc/entry/troubleshooting_connection_problems_in_jconsole
# for more on configuring JMX through firewalls, etc. (Short version:
# get it working with no firewall first.)
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT"
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl=false"
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.password.file=/etc/cassandra/jmxremote.password"
JVM_OPTS="$JVM_OPTS $JVM_EXTRA_OPTS"
What are the values of MAX_HEAP_SIZE, HEAP_NEWSIZE, CASSANDRA_MEMORY_IN_MB?5. /etc/dse/spark/spark-env.sh
#!/usr/bin/env bash
export SPARK_MASTER_PORT=7077
export SPARK_MASTER_WEBUI_PORT=7080
export SPARK_WORKER_WEBUI_PORT=7081
# The hostname or IP address Cassandra rpc/native protocol is bound to:
# SPARK_CASSANDRA_CONNECTION_HOST="127.0.0.1"
if [ $SPARK_CASSANDRA_CONNECTION_HOST ]; then
SPARK_CASSANDRA_CONNECTION_HOST_PARAM="-Dspark.cassandra.connection.host=$SPARK_CASSANDRA_CONNECTION_HOST"
else
SPARK_CASSANDRA_CONNECTION_HOST_PARAM="-Dspark.cassandra.connection.host=$CASSANDRA_ADDRESS" # defined in dse.in.sh
fi
# The hostname or IP address for the driver to listen on. If there is more network interfaces you
# can specify which one is to be used by Spark Shell or other Spark applications.
# export SPARK_DRIVER_HOST="127.0.0.1"
if [ $SPARK_DRIVER_HOST ]; then
SPARK_DRIVER_HOST_PARAM="-Dspark.driver.host=$SPARK_DRIVER_HOST"
else
SPARK_DRIVER_HOST_PARAM=""
fi
# The default amount of memory used by a single executor (also, for the executor used by Spark REPL).
# It can be modified in particular application by command line parameters. 512M is the default value.
export SPARK_EXECUTOR_MEMORY="1024M"
# The default number of cores assigned to each application. It can be modified in particular application. If left blank,
# each Spark application will consume all available cores in the cluster.
export DEFAULT_PER_APP_CORES="3"
if [ $DEFAULT_PER_APP_CORES ]; then
DEFAULT_PER_APP_CORES_PARAM="-Dspark.deploy.defaultCores=$DEFAULT_PER_APP_CORES"
else
DEFAULT_PER_APP_CORES_PARAM=""
fi
# Set the amount of memory used by Spark Worker - if uncommented, it overrides the setting initial_spark_worker_resources in dse.yaml.
export SPARK_WORKER_MEMORY=4096m
# Set the number of cores used by Spark Worker - if uncommented, it overrides the setting initial_spark_worker_resources in dse.yaml.
export SPARK_WORKER_CORES=2
# The amount of memory used by SparkShell in the client environment.
export SPARK_REPL_MEM="256M"
# The amount of memory used by Spark Driver program
export SPARK_DRIVER_MEMORY="512M"
# Directory for Spark temporary files. It will be used by Spark Master, Spark Worker, Spark Shell and Spark applications.
export SPARK_TMP_DIR="/tmp/spark"
# Directory where RDDs will be cached
export SPARK_RDD_DIR="/var/lib/spark/rdd"
# The directory for storing master.log and worker.log files
export SPARK_LOG_DIR="/var/log/spark"
# These options are common to all Spark related daemons and applications
export SPARK_COMMON_OPTS="$SPARK_COMMON_OPTS $SPARK_CASSANDRA_CONNECTION_HOST_PARAM -Dspark.kryoserializer.buffer.mb=10 "
# Individual Java opts for different Spark components
#
# Warning: Be careful when changing temporary subdirectories. Make sure they different for different Spark components
# and they are set with spark.local.dir for Spark Master and Spark Worker, and with java.io.tmpdir for Spark executor,
# Spark shell(repl) and Spark applications. Jobs may not finish properly (hang) if temporary directories overlap.
#
# Warning: When changing temporary or logs locations, consider permissions and ownership of files created by particular
# Spark components. Wrongly specified directories here may result in security related errors.
export SPARK_MASTER_OPTS=" $DEFAULT_PER_APP_CORES_PARAM -Dspark.local.dir=$SPARK_TMP_DIR/master -Dlog4j.configuration=file://$SPARK_CONF_DIR/log4j-server.properties -Dspark.log.file=$SPARK_LOG_DIR/master.log "
export SPARK_WORKER_OPTS=" -Dspark.local.dir=$SPARK_TMP_DIR/worker -Dlog4j.configuration=file://$SPARK_CONF_DIR/log4j-server.properties -Dspark.log.file=$SPARK_LOG_DIR/worker.log "
export SPARK_EXECUTOR_OPTS=" -Djava.io.tmpdir=$SPARK_RDD_DIR -Dlog4j.configuration=file://$SPARK_CONF_DIR/log4j-executor.properties "
export SPARK_REPL_OPTS=" -Djava.io.tmpdir=$SPARK_TMP_DIR/$USER $SPARK_DRIVER_HOST_PARAM "
export SPARK_SUBMIT_OPTS=" -Djava.io.tmpdir=$SPARK_TMP_DIR/$USER $SPARK_DRIVER_HOST_PARAM "
# Directory to run applications in, which will include both logs and scratch space (default: /var/lib/spark/work).
export SPARK_WORKER_DIR="/var/lib/spark/work"
# Temporary storage location (as of Spark 1.0)
export SPARK_LOCAL_DIRS="$SPARK_RDD_DIR"
No comments:
Post a Comment