Storing log data with Loki

apt-get install curl gnupg2 ca-certificates lsb-release debian-archive-keyring

curl --silent https://apt.grafana.com/gpg.key | 
  gpg --dearmor --yes --output /etc/apt/keyrings/grafana.gpg

cat << EOF > /etc/apt/sources.list.d/apt-grafana.sources
X-Repolib-Name: APT Grafana
Types: deb
URIs: https://apt.grafana.com
Suites: stable
Components: main
Architectures: amd64
Signed-By: /etc/apt/keyrings/grafana.gpg
Enabled: yes
EOF

echo -e "Package: *\nPin: origin apt.grafana.com\nPin: release o=stable\nPin-Priority: 900\n" \
  | tee /etc/apt/preferences.d/99loki

export LOKI_VERSION="3.5.1"
apt-get update
apt-get install loki=${LOKI_VERSION} redis

openssl rand -base64 25 | tr --delete /=+ | cut --characters -32

systemctl restart redis

systemctl restart loki

# Run all components in single binary mode
target: all

# Disable authentication
auth_enabled: false

# Optionally, disable reporting statistics to stats.grafana.org
analytics:
  reporting_enabled: false

# Configuration options of the Loki server.
# See: https://grafana.com/docs/loki/latest/configure/#server
# Log level is set to `info` by default.
server:

  # Optionally, change the default ports
  http_listen_port: 8080
  grpc_listen_port: 8081

  # Default listen addresses
  http_listen_address: "0.0.0.0"
  grpc_listen_address: "0.0.0.0"

  # TLS configuration over HTTP
  # All gRPC connections will use the localhost, so no TLS needed
  tls_min_version: VersionTLS12
  http_tls_config:
    cert_file: "/etc/ssl/certs/localdomain.com.crt"
    key_file: "/etc/ssl/private/localdomain.com.key"

  # Log level
  log_level: info

  # Default size limit of gRPC messages this server can receive and send (4 MB)
  grpc_server_max_recv_msg_size: 4194304
  grpc_server_max_send_msg_size: 4194304

  # Increase the limit on the number of concurrent streams for gRPC
  # calls per client connection from 100 to 1K
  grpc_server_max_concurrent_streams: 1000

# Common configuration to be shared between multiple modules
# See: https://grafana.com/docs/loki/latest/configure/#common
common:
  path_prefix: /var/lib/loki
  storage:
    # Use the local filesystem as storage
    filesystem:
      # Directory to store chunks in
      chunks_directory: /var/lib/loki/chunks
      # Directory to store rules in
      rules_directory: /var/lib/loki/rules
  
  # Instruct components to use the localhost to communicate among themselves
  instance_addr: 127.0.0.1

  # Adapt the default replication factor to a single-instance deploy
  replication_factor: 1

  # Use the in-memory key/value store
  ring:
    kvstore:
      store: inmemory

# Configuration options for query splitting and caching in the query-frontend
query_range:
  
  # Cache backend for the query frontend component
  results_cache:
  
    # Configures the cache backend for a specific component.
    # See: https://grafana.com/docs/loki/latest/configure/#cache_config
    cache:
  
      # Default validity of entries (unless overridden)
      default_validity: 1h

      # Redis server configuration
      redis:
        # Redis endpoint
        endpoint: 127.0.0.1:6379
        # Let Loki manage TTL via default validity
        expiration: 0s
        # Let Loki auto-scale (10 per CPU core)
        pool_size: 0
        # Password to use when connecting to Redis
        password: <password>
        # Keep idle connections open indefinitely
        idle_timeout: 0s
        # Keep connections alive forever
        max_connection_age: 0s

    # Use compression in cache
    compression: snappy

  # Cache query results (default is false)
  cache_results: true

  # Default behaviour to cache index stats, volume, series and label  query results
  cache_index_stats_results: true
  cache_volume_results: true
  cache_series_results: true
  cache_label_results: true

# Configure global limits
# See: https://grafana.com/docs/loki/latest/configure/#limits_config
limits_config:

  # A. Ingestion and rate limit

  # Enforce limits globally
  ingestion_rate_strategy: "global"

  # Default per-user ingestion rate limit in sample size per second (4MB)
  ingestion_rate_mb: 4

  # Default per-user allowed ingestion burst size (6MB). Burst size is the per-distributor
  # local rate limiter. Should be aligned with the expected max payload size per push request.
  ingestion_burst_size_mb: 6

  # Default maximum byte rate per second per stream (3MB)
  per_stream_rate_limit: "3MB"

  # Default maximum burst bytes per stream (15MB)
  per_stream_rate_limit_burst: "15MB"

  # B. Retention and aging

  # Default behaviour to reject samples meeting the criteria below
  reject_old_samples: true

  # Default maximum accepted sample age
  reject_old_samples_max_age: "1w"
  
  # Duration which table will be created/deleted before/after it is needed.
  # Samples from before this time will not be accepted.
  creation_grace_period: "10m"

  # Retention period to apply to stored data. Must be either 0 (disabled) or a multiple of 24h.
  # Requires `compactor.retention_enabled = true`. 43800 hours equals 5 years.
  retention_period: "43800h"

  # Default deletion mode
  deletion_mode: "filter-and-delete"

  # C. Query and performance

  # Enable metric aggregation to speed up histogram queries
  metric_aggregation_enabled: true

  # Default maximum number of unique series that is returned by a metric query
  max_query_series: 500

  # Default maximum number of queries that will be scheduled in parallel by the frontend
  max_query_parallelism: 32

  # Default limit to length of chunk store queries
  max_query_length: 30d1h

  # Default (lack of) limit to the length of the [range] inside a range query
  max_query_range: 0s

  # D. Metadata and features

  # Do not attempt auto-discovery of log levels during ingestion.
  # We will do this manually, via regex within Promtail.
  discover_log_levels: false

  # Default maximum number of active streams per user, across the cluster
  max_global_streams_per_user: 5000

  # Default behaviour to allow user to send structured metadata in push payload
  allow_structured_metadata: true

  # Default behaviour to enable log-volume endpoints for log volume dashboards.
  # Allows aggregating log counts (or sizes) over specified time ranges, grouped by labels,
  # e.g., "logs per minute" trend.
  volume_enabled: true

# Configuration options of the compactor.
# See: https://grafana.com/docs/loki/latest/configure/#compactor
compactor:
  # Activate custom (per-stream, per-tenant) retention
  retention_enabled: true

  # Default aximum number of tables to compact in parallel
  max_compaction_parallelism: 1

  # Default number of upload/remove operations to execute in parallel when finalizing
  # a compaction, per compact operation, which can be executed in parallel
  upload_parallelism: 10

  # Define a store to be used for managing delete requests (default is none, which is invalid)
  # Requires setting `storage_config.filesystem.directory`.
  delete_request_store: filesystem

  # Default max number of delete requests to run per compaction cycle
  delete_batch_size: 70

  # Default interval at which compaction operations are run (how often)
  compaction_interval: 10m

  # Default delay after which chunks will be fully deleted during retention
  retention_delete_delay: 2h

  # Default working directory to store downloaded bloom blocks
  working_directory: "/var/lib/loki/compactor"

# Configuration options of the ingester.
# See: https://grafana.com/docs/loki/latest/configure/#ingester
ingester:

  # Flush chunks after 24h to prevent too many small files
  max_chunk_age: 24h

  # Align with `max_chunk_age` to flush inactive streams
  chunk_idle_period: 24h

  # Increase target compressed size (2MB) for chunks to balance file count and query performance.
  chunk_target_size: 2097152

  # Default target uncompressed size (256KB) of a chunk block lets ZFS compression handle this
  # effficiently. When this threshold is exceeded, the head block will be cut and compressed
  # inside the chunk.
  chunk_block_size: 262144

  # Retain chunks in memory after they have been flushed for late-arriving logs
  chunk_retain_period: 15m

  # Default maximum retries for failed flushes, critical for filesystem reliability
  max_retries: 10

  # How the lifecycle of the ingester will operate
  lifecycler:
    ring:

      # Default period at which to heartbeat to the ring
      heartbeat_period: 15s

      # Default heartbeat timeout after which compactors are considered unhealthy within the ring
      heartbeat_timeout: 1m

    # Default duration to sleep before exiting on shutdown (no data loss with WAL)
    final_sleep: 0s

# Configuration options of the schema.
# See: https://grafana.com/docs/loki/latest/configure/#schema_config
# See: https://grafana.com/docs/loki/latest/configure/#period_config
schema_config:
  
  # Configures the chunk index schema and where it is stored.
  configs:
 
    # Configure what index schemas should be used for from specific time periods
    # The date of the first day that index buckets should be created
    - from: "2024-01-01"
 
      # Which index to use. Either 'tsdb' or 'boltdb-shipper'.  <store>
      # and <object_store> below affect which <storage_config> key is used.
      store: tsdb
      # Which store to use for the chunks
      object_store: filesystem
      # The schema version to use (current recommended schema is v13).
      schema: v13
  
      # Configures how the index is updated and stored
      index:
        # Path prefix for index tables. Prefix always needs to end with a path
        # delimiter '/', except when the prefix is empty (default is "index/").
        path_prefix: "index/"
        # Table prefix for all period tables (default is "")
        prefix: "index_"
        # Table period. Retention is only available if the index period is 24h.
        # Single store TSDB and single store BoltDB require 24h index period.
        period: "24h"

      # Configure how the chunks are updated and stored
      chunks:
        # Table prefix for all period tables
        prefix: "chunks_"
        # Table period
        period: "24h"

      # Default number of shards that will be created is fine unless high cardinality is
      # encountered (many unique label combinations).
      row_shards: 16

# Configuration options of the pattern ingester.
# See: https://grafana.com/docs/loki/latest/configure/#pattern-ingester
pattern_ingester:

  # Enable the pattern ingester
  enabled: true

  # Metric aggregation settings
  metric_aggregation:

    # Address of the Loki instance to push aggregated metrics to,
    # matching `server.http_listen_address` and `server.http_listen_port`
    loki_address: 127.0.0.1:8080

    # HTTP client tuning
    http_client_config:
      enable_http2: true
      proxy_from_environment: false
      # Fail fast on slow pushes
      timeout: 10s
      # Disabled for security
      follow_redirects: false

    # Reduce load by incresing the default value (30s) of the interval between operations
    pull_push_interval: 2m

  # Configures how the lifecycle of the pattern ingester will operate and where
  # it will register for discovery.
  lifecycler:
    # `ring` inherits from `common.ring` (in-memory)
    # Default period at which to heartbeat
    heartbeat_period: 5s
    # Default heartbeat timeout after which instance is assumed to be unhealthy
    heartbeat_timeout: 1m    

# Configuration options for the bloom planner and builder servers
bloom_build:

  # Enable the experimental feature
  enabled: true

  # Limit concurrency so two bloom‑build tasks run at once
  max_builders: 2
  # Retry each individual bloom‑build task twice on failure before marking it as failed
  task_max_retries: 2
  # How long the planner will wait for a builder to acknowledge or complete a task before requeuing
  builder_response_timeout: 30s
  # ZFS already compresses disk storage, but this helps in-memory transfer efficiency and page size
  bloom_block_encoding: "zstd"
  # Once a new bloom block is built, fetch it into the gateway's cache so queries can use it without extra latency
  prefetch_blocks: true

  # Decrease the default to 150MB to keep down the heap footprint of each builder task.
  # Smaller blocks give the builder more, finer‑grained units of work.
  # In a single‑instance environment, this helps distribute CPU and I/O more evenly across cores.
  max_block_size: 150MiB
  # Maximum size of the raw bloom filter per individual time series. Filters larger than this are dropped.
  max_bloom_size: 64MiB
  # Instruct the planner to break the series key‑space into 128 slices so builds can run in parallel,
  # smaller units, aiming for roughly 8 GB of series data per bloom‑build task.
  planning_strategy: "split_keyspace_by_factor"
  split_keyspace_by: 128
  split_target_series_chunk_size: 8GB

  # Configuration options for the bloom planner server
  planner:

    # Running the bloom creation planning twice a day is enough
    planning_interval: 12h

    # Cover recent activity and include recent historical context.
    # Setting `min_table_offset`to 0 ensures that the current active index table (where most
    # writes happen) is included, which enables faster recent queries.
    # Setting `max_table_offset` to 2 means the planner includes the last two inactive tables
    # as well, which helps with queries that span recent time windows (e.g., last 24–48h).
    # Because older tables rarely change, there is little benefit in trying to replan them
    # every time unless retention requirements change.
    min_table_offset: 0
    max_table_offset: 2

    retention:
      # Enable bloom retention
      enabled: true
    queue:
      # Reduce the default maximum number of tasks to queue per tenant to keep the queue size
      # modest, reducing the change of OOM and GC pressure in our single instance. Even under
      # heavy indexing, 1000 pending bloom‑build tasks provides ample headroom.
      max_queued_tasks_per_tenant: 1000
      # Move queued tasks off the heap and onto the ZFS filesystem to avoi bloating memory.
      # ZFS has built-in compression, which makes on-disk queues very space-efficient.
      store_tasks_on_disk: true
      # Directory to store tasks on disk.
      tasks_disk_directory: /var/lib/loki/bloom-planner-queue
      # Default setting not to clean the tasks directory upon startup.
      clean_tasks_directory: false

  # Configuration options for the bloom builder server
  builder:
    # Address and port of the bloom planner,
    # matching `server.grpc_listen_address` and `server.grpc_listen_port`.
    planner_address: 127.0.0.1:8081

    # See: https://grafana.com/docs/loki/latest/configure/#grpc_client
    grpc_config:
      # Decrease the maximum size of messages from 100MB to 10MB to prevent a single huge
      # bloom‑filter transfer from triggering a GC stall or OOM in our single-instance deploy
      # and to align with expected bloom-block sizes
      max_send_msg_size: 10485760
      max_recv_msg_size: 10485760

    # Controls how the builder retries when an individual bloom‑build task (its gRPC call to 
    # the planner) fails temporarily, e.g., due to CPU contention, or I/O stall).
    # It implements an exponential‑style retry delay so that the system:
    # 1. Doesn’t hammer the planner continuously on every failure
    # 2. Gives transient issues time to clear before trying again.
    # 3. Eventually gives up (after `max_retries`) 
    backoff_config:
      # These backoff settings keep bloom‑build tasks responsive to transient hiccups while 
      # ensuring fail‑fast behavior for persistent problems—ideal for a single‑instance

      # Prevents hot‑looping on very rapid retries (100 ms can be too tight under load).
      # A slightly longer base delay gives transient pressure (e.g. brief CPU spikes or 
      # filesystem contention) time to settle.
      min_period: 250ms
      # Caps backoff so that, in persistent error scenarios, we don’t end up waiting a 
      # full 10 s before retrying. A 5 s ceiling balances giving the system breathing 
      # space versus surfacing issues faster.
      max_period: 5s
      # Halves the default retry count so that truly failing tasks will give up sooner,
      # free up queue slots, and let you detect and investigate root causes, rather than 
      # thrashing on repeated futile retries.
      max_retries: 5

bloom_gateway:

  # Turn the gateway on so queries can use Bloom filters to skip irrelevant chunks
  # and actually apply the bloom‑based chunk filtering in the read path
  enabled: true
  enable_filtering: true

  # Number of parallel workers testing chunks against Bloom filters
  worker_concurrency: 2
  # Lower the number of blocks each worker can check at once to reduce peak memory and gRPC streams
  block_query_concurrency: 4
  # Reduce the size of the in-flight Bloom-filter checks per tenant queue to keep memory bounded
  # and back-presure predictable in our single-instance.
  max_outstanding_per_tenant: 256
  # Redice the chunk-filter tasks to batch together to reduce per-RPC payload size, easing gRPC
  # backpressure and GC presure.
  num_multiplex_tasks: 128

  client:
    # Point the gateway's gRPC client at the planner endpoint,
    # matching `server.grpc_listen_address` and `server.grpc_listen_port`.
    addresses: 127.0.0.1:8081
    pool_config:
      # Decreas how often the client refreshes its list of gateway backends, to reduce DNS or
      # service-discovery churn on a stable, single-node address list.
      check_interval: 30s
    grpc_client_config:
      # Cap gRPC message sizes to 10 MB to bound per-RPC memory (from 100 MB) to reduce chances of
      # triggering GC spikes if a single RPC returns many blocks.
      max_send_msg_size: 10MiB
      max_recv_msg_size: 10MiB

# Top‑level storage config for the ruler (replaces deprecated `ruler.storage`)
ruler_storage:
  backend: filesystem
  local:
    # Where rule files in YAML format will be stored
    directory: /var/lib/loki/rules

ruler:
  # Do not expose the alert‑rule API until Alertmanager is in play
  enable_api: false

frontend:

  # Log slow queries
  log_queries_longer_than: 500ms

  # Include these request headers in slow‑query logs
  log_query_request_headers: "User‑Agent,X‑Request‑ID"

  # Enable per‑query stats
  query_stats_enabled: true

  # Bump max body size of requests from 10 MB to 20 MB
  max_body_size: 20971520

  # Throttle concurrent outstanding queries per tenant to bound memory,
  # improving efficiency of ZFS ARC, if present
  max_outstanding_per_tenant: 512

  # Increase the time between DNS lookups of the scheduler
  scheduler_dns_lookup_period: 30s

  # Reduce the time to wait for in‑flight queries on shutdown
  graceful_shutdown_timeout: 2m

# journalctl --unit=loki --follow
level=info ts=2025-05-12T14:50:02.123Z caller=frontend.go:XXX msg="slow query detected" query_duration=611.342ms method=GET path=/loki/api/v1/query_range params="..." headers="User-Agent=Grafana/X.X"

querier:
  # Adapt the number of concurrent queries for 2-4 CPU cores
  max_concurrent: 4

  # Decrease message size from 100 MiB to 10 MiB to bound memory per-RPC
  grpc_client_config:
    max_send_msg_size: 10485760
    max_recv_msg_size: 10485760

  # Default values of the engine, for reference
  engine:
    # Cap how far an instant query looks back
    max_look_back_period: 30s
    # topk sketch size
    max_count_min_sketch_heap_size: 10000

  # Default values regarding multi-tenancy, for reference
  # Read both ingesters & store
  query_store_only: false
  # Skip store if object store is unavailable
  query_ingester_only: false
  # Single-tenant mode
  multi_tenant_queries_enabled: false
  # No header-based overrides
  per_request_limits_enabled: false
  # No partitioning of ingesters
  query_partition_ingesters: false

  # Default values of query size and volume limits, for reference
  # Prevent runaway chunk scans
  max_chunks_per_query: 2000000
  # Cap series returned
  max_query_series: 500
  # Allow going all the way back in time when querying data and metadata
  max_query_lookback: 0s
  # Maximum span of a single query
  max_query_length: 30d1h
  # Do not limit the length of the range in range queries
  max_query_range: 0s
  # Maximum number of queries that will be scheduled in parallel by the frontend
  max_query_parallelism: 32
  # Maximum number of queries will be scheduled in parallel by the frontend for TSDB schemas.
  tsdb_max_query_parallelism: 128
  # Target maximum number of bytes assigned to a single sharded query
  tsdb_max_bytes_per_shard: 600MB
  # Sharding strategy to use in query planning
  tsdb_sharding_strategy: "power_of_two"

  # Default values of frontend and scheduler integration, for reference. When neither
  # `frontend_address` nor `scheduler_address` are set, queries are only received via HTTP endpoint.
  frontend_address: ""
  scheduler_address: ""

query_range:
  # Keep raw time bounds, i.e., do not mutate incoming queries to align their
  # start and end with their step (this is the default behaviour).
  align_queries_with_step: false
  # Do not attempt to parallelise query execution via a shared scheduler (default behaviour).
  parallelise_shardable_queries: false

  # Time-range results cache
  cache_results: true
  results_cache:
    cache:
      redis:
        endpoint: 127.0.0.1:6379
        password: <password>
        expiration: 1h
        pool_size: 10
        idle_timeout: 30s
        max_connection_age: 5m
    compression: snappy

  # Index‐stats cache
  cache_index_stats_results: true
  index_stats_results_cache:
    cache:
      redis:
        endpoint: 127.0.0.1:6379
        password: <password>
        # Fresher index counts
        expiration: 15m
        pool_size: 5
        idle_timeout: 30s
        max_connection_age: 5m
    compression: snappy

  # Volume‐results cache
  cache_volume_results: true
  volume_results_cache:
    cache:
      redis:
        endpoint: 127.0.0.1:6379
        password: <password>
        expiration: 1h
        pool_size: 5
        idle_timeout: 30s
        max_connection_age: 5m
    compression: snappy

  # Series‐results cache
  cache_series_results: true
  series_results_cache:
    cache:
      redis:
        endpoint: 127.0.0.1:6379
        password: <password>
        expiration: 1h
        pool_size: 5
        idle_timeout: 30s
        max_connection_age: 5m
    compression: snappy

  # Label‐values cache
  cache_label_results: true
  label_results_cache:
    cache:
      redis:
        endpoint: 127.0.0.1:6379
        password: <password>
        # Labels change less often
        expiration: 2h
        pool_size: 5
        idle_timeout: 30s
        max_connection_age: 5m
    compression: snappy

  # Instant‐metric cache
  cache_instant_metric_results: true
  instant_metric_results_cache:
    cache:
      redis:
        endpoint: 127.0.0.1:6379
        password: <password>
        expiration: 30m
        pool_size: 5
        idle_timeout: 30s
        max_connection_age: 5m
    compression: snappy