From 2c76da28683e6d6e4365e87493b2ea4d07441007 Mon Sep 17 00:00:00 2001 From: jgilaber Date: Mon, 24 Mar 2025 16:23:57 +0100 Subject: [PATCH] Make prometheus the default devstack example Change the devstack local.conf samples and devstack multinode contributor doc to demonstrate deploying watcher with prometheus as datasource instead of gnocchi. Keep the gnocchi as an alternative deployment example. Depends-On: https://review.opendev.org/c/openstack/watcher/+/946230 Depends-On: https://review.opendev.org/c/openstack/devstack-plugin-prometheus/+/946254 Change-Id: I721b550a03f9e5350a3f1ab10292faa1c50049a7 --- devstack/local.conf.compute | 6 ++- devstack/local.conf.controller | 54 ++++++++++++++++++++++- devstack/local_gnocchi.conf.compute | 53 ++++++++++++++++++++++ devstack/local_gnocchi.conf.controller | 57 ++++++++++++++++++++++++ devstack/prometheus.yml | 16 +++++++ doc/source/contributor/devstack.rst | 61 +++++++++++++++++++++++--- 6 files changed, 237 insertions(+), 10 deletions(-) create mode 100644 devstack/local_gnocchi.conf.compute create mode 100644 devstack/local_gnocchi.conf.controller create mode 100644 devstack/prometheus.yml diff --git a/devstack/local.conf.compute b/devstack/local.conf.compute index c7fb54279..46127ce7f 100644 --- a/devstack/local.conf.compute +++ b/devstack/local.conf.compute @@ -26,7 +26,7 @@ GLANCE_HOSTPORT=${SERVICE_HOST}:9292 DATABASE_TYPE=mysql # Enable services (including neutron) -ENABLED_SERVICES=n-cpu,n-api-meta,c-vol,q-agt,placement-client +ENABLED_SERVICES=n-cpu,n-api-meta,c-vol,q-agt,placement-client,node-exporter NOVA_VNC_ENABLED=True NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_auto.html" @@ -42,6 +42,10 @@ disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api LOGFILE=$DEST/logs/stack.sh.log LOGDAYS=2 +CEILOMETER_BACKEND="none" +CEILOMETER_BACKENDS="none" +enable_plugin devstack-plugin-prometheus https://opendev.org/openstack/devstack-plugin-prometheus + [[post-config|$NOVA_CONF]] [DEFAULT] compute_monitors=cpu.virt_driver diff --git a/devstack/local.conf.controller b/devstack/local.conf.controller index eb46f63d0..df7363c3c 100644 --- a/devstack/local.conf.controller +++ b/devstack/local.conf.controller @@ -8,6 +8,11 @@ DATABASE_PASSWORD=stackdb RABBIT_PASSWORD=stackqueue SERVICE_PASSWORD=$ADMIN_PASSWORD SERVICE_TOKEN=azertytoken +# Disable sg_core prometheus config copy +PROMETHEUS_ENABLE=false +# PROMETHEUS_CONFIG_FILE var conflicts with sg_core var +# to avoid issue, set PROMETHEUS_CONF_DIR +PROMETHEUS_CONF_DIR="${DEST}" HOST_IP=192.168.42.1 # Change this to your controller node IP address #HOST_IPV6=2001:db8::7 @@ -18,6 +23,10 @@ NETWORK_GATEWAY=10.254.1.1 # Change this for your network MULTI_HOST=1 +CEILOMETER_ALARM_THRESHOLD="6000000000" +CEILOMETER_BACKENDS="sg-core" +CEILOMETER_PIPELINE_INTERVAL="15" + #Set this to FALSE if do not want to run watcher-api behind mod-wsgi #WATCHER_USE_MOD_WSGI=TRUE @@ -40,8 +49,10 @@ disable_service ceilometer-acompute # Enable the ceilometer api explicitly(bug:1667678) enable_service ceilometer-api -# Enable the Gnocchi plugin -enable_plugin gnocchi https://github.com/gnocchixyz/gnocchi +enable_service prometheus +enable_plugin aodh https://opendev.org/openstack/aodh +enable_plugin devstack-plugin-prometheus https://opendev.org/openstack/devstack-plugin-prometheus +enable_plugin sg-core https://github.com/openstack-k8s-operators/sg-core main LOGFILE=$DEST/logs/stack.sh.log LOGDAYS=2 @@ -55,3 +66,42 @@ compute_monitors=cpu.virt_driver # can change this to just versioned when ceilometer handles versioned # notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449 notification_format=both + +[[post-config|$WATCHER_CONF]] +[prometheus_client] +host = 127.0.0.1 +port = 9090 + +[watcher_cluster_data_model_collectors.baremetal] +period = 120 + +[watcher_cluster_data_model_collectors.compute] +period = 120 + +[watcher_cluster_data_model_collectors.storage] +period = 120 + +[watcher_datasources] +datasources = prometheus + +[[test-config|$TEMPEST_CONFIG]] +[optimize] +datasource = prometheus + +[service_available] +sg_core = True + +[telemetry] +ceilometer_polling_interval = 15 +disable_ssl_certificate_validation = True + +[telemetry_services] +metric_backends = prometheus + +[compute] +min_compute_nodes = 2 +min_microversion = 2.56 + +[compute-feature-enabled] +block_migration_for_live_migration = True +live_migration = True diff --git a/devstack/local_gnocchi.conf.compute b/devstack/local_gnocchi.conf.compute new file mode 100644 index 000000000..c7fb54279 --- /dev/null +++ b/devstack/local_gnocchi.conf.compute @@ -0,0 +1,53 @@ +# Sample ``local.conf`` for compute node for Watcher development +# NOTE: Copy this file to the root DevStack directory for it to work properly. + +[[local|localrc]] + +ADMIN_PASSWORD=nomoresecrete +DATABASE_PASSWORD=stackdb +RABBIT_PASSWORD=stackqueue +SERVICE_PASSWORD=$ADMIN_PASSWORD +SERVICE_TOKEN=azertytoken + +HOST_IP=192.168.42.2 # Change this to this compute node's IP address +#HOST_IPV6=2001:db8::7 +FLAT_INTERFACE=eth0 + +FIXED_RANGE=10.254.1.0/24 # Change this to whatever your network is +NETWORK_GATEWAY=10.254.1.1 # Change this for your network + +MULTI_HOST=1 + +SERVICE_HOST=192.168.42.1 # Change this to the IP of your controller node +MYSQL_HOST=$SERVICE_HOST +RABBIT_HOST=$SERVICE_HOST +GLANCE_HOSTPORT=${SERVICE_HOST}:9292 + +DATABASE_TYPE=mysql + +# Enable services (including neutron) +ENABLED_SERVICES=n-cpu,n-api-meta,c-vol,q-agt,placement-client + +NOVA_VNC_ENABLED=True +NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_auto.html" +VNCSERVER_LISTEN=0.0.0.0 +VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP # or HOST_IPV6 + +NOVA_INSTANCES_PATH=/opt/stack/data/instances + +# Enable the Ceilometer plugin for the compute agent +enable_plugin ceilometer https://opendev.org/openstack/ceilometer +disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api + +LOGFILE=$DEST/logs/stack.sh.log +LOGDAYS=2 + +[[post-config|$NOVA_CONF]] +[DEFAULT] +compute_monitors=cpu.virt_driver +[notifications] +# Enable both versioned and unversioned notifications. Watcher only +# uses versioned notifications but ceilometer uses unversioned. We +# can change this to just versioned when ceilometer handles versioned +# notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449 +notification_format=both diff --git a/devstack/local_gnocchi.conf.controller b/devstack/local_gnocchi.conf.controller new file mode 100644 index 000000000..eb46f63d0 --- /dev/null +++ b/devstack/local_gnocchi.conf.controller @@ -0,0 +1,57 @@ +# Sample ``local.conf`` for controller node for Watcher development +# NOTE: Copy this file to the root DevStack directory for it to work properly. + +[[local|localrc]] + +ADMIN_PASSWORD=nomoresecrete +DATABASE_PASSWORD=stackdb +RABBIT_PASSWORD=stackqueue +SERVICE_PASSWORD=$ADMIN_PASSWORD +SERVICE_TOKEN=azertytoken + +HOST_IP=192.168.42.1 # Change this to your controller node IP address +#HOST_IPV6=2001:db8::7 +FLAT_INTERFACE=eth0 + +FIXED_RANGE=10.254.1.0/24 # Change this to whatever your network is +NETWORK_GATEWAY=10.254.1.1 # Change this for your network + +MULTI_HOST=1 + + +#Set this to FALSE if do not want to run watcher-api behind mod-wsgi +#WATCHER_USE_MOD_WSGI=TRUE + +# This is the controller node, so disable nova-compute +disable_service n-cpu + +# Enable the Watcher Dashboard plugin +enable_plugin watcher-dashboard https://opendev.org/openstack/watcher-dashboard + +# Enable the Watcher plugin +enable_plugin watcher https://opendev.org/openstack/watcher + +# Enable the Ceilometer plugin +enable_plugin ceilometer https://opendev.org/openstack/ceilometer + +# This is the controller node, so disable the ceilometer compute agent +disable_service ceilometer-acompute + +# Enable the ceilometer api explicitly(bug:1667678) +enable_service ceilometer-api + +# Enable the Gnocchi plugin +enable_plugin gnocchi https://github.com/gnocchixyz/gnocchi + +LOGFILE=$DEST/logs/stack.sh.log +LOGDAYS=2 + +[[post-config|$NOVA_CONF]] +[DEFAULT] +compute_monitors=cpu.virt_driver +[notifications] +# Enable both versioned and unversioned notifications. Watcher only +# uses versioned notifications but ceilometer uses unversioned. We +# can change this to just versioned when ceilometer handles versioned +# notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449 +notification_format=both diff --git a/devstack/prometheus.yml b/devstack/prometheus.yml new file mode 100644 index 000000000..a8520cbbe --- /dev/null +++ b/devstack/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 10s +scrape_configs: + - job_name: "node" + static_configs: + - targets: ["controller:3000"] + - targets: ["controller:9100"] + labels: + fqdn: "controller" # change the hostname here to your controller hostname + - targets: ["compute-1:9100"] + labels: + fqdn: "compute-1" # change the hostname here to your fist compute hostname + - targets: ["compute-2:9100"] + labels: + fqdn: "compute-2" # change the hostname her to your secondd compute hostname + # add as many blocks as compute nodes you have diff --git a/doc/source/contributor/devstack.rst b/doc/source/contributor/devstack.rst index e7badb40d..ca3e36358 100644 --- a/doc/source/contributor/devstack.rst +++ b/doc/source/contributor/devstack.rst @@ -31,14 +31,45 @@ Quick Devstack Instructions with Datasources ============================================ Watcher requires a datasource to collect metrics from compute nodes and -instances in order to execute most strategies. To enable this a -``[[local|localrc]]`` to setup DevStack for some of the supported datasources -is provided. These examples specify the minimal configuration parameters to -get both Watcher and the datasource working but can be expanded is desired. +instances in order to execute most strategies. To enable this two possible +examples of ``[[local|localrc]]`` to setup DevStack for some of the +supported datasources is provided. These examples specify the minimal +configuration parameters to get both Watcher and the datasource working +but can be expanded is desired. +The first example configures watcher to user prometheus as a datasource, while +the second example show how to use gnocchi as the datasource. The procedure is +equivalent, it just requires using the ``local.conf.controller`` and +``local.conf.compute`` in the first example and +``local_gnocchi.conf.controller`` and ``local_gnocchi.conf.compute`` in the +second. + +Prometheus +---------- + +With the Prometheus datasource most of the metrics for compute nodes and +instances will work with the provided configuration but metrics that +require Ironic such as ``host_airflow and`` ``host_power`` will still be +unavailable as well as ``instance_l3_cpu_cache`` + +.. code-block:: ini + + [[local|localrc]] + + enable_plugin watcher https://opendev.org/openstack/watcher + enable_plugin watcher-dashboard https://opendev.org/openstack/watcher-dashboard + enable_plugin ceilometer https://opendev.org/openstack/ceilometer.git + enable_plugin aodh https://opendev.org/openstack/aodh + enable_plugin devstack-plugin-prometheus https://opendev.org/openstack/devstack-plugin-prometheus + enable_plugin sg-core https://github.com/openstack-k8s-operators/sg-core main + + + CEILOMETER_BACKEND=sg-core + [[post-config|$NOVA_CONF]] + [DEFAULT] + compute_monitors=cpu.virt_driver Gnocchi ------- - With the Gnocchi datasource most of the metrics for compute nodes and instances will work with the provided configuration but metrics that require Ironic such as ``host_airflow and`` ``host_power`` will still be @@ -96,7 +127,8 @@ Detailed DevStack Instructions cd ~ git clone https://opendev.org/openstack/devstack.git -#. For each compute node, copy the provided `local.conf.compute`_ example file +#. For each compute node, copy the provided `local.conf.compute`_ + (`local_gnocchi.conf.compute`_ if deploying with gnocchi) example file to the compute node's system at ~/devstack/local.conf. Make sure the HOST_IP and SERVICE_HOST values are changed appropriately - i.e., HOST_IP is set to the IP address of the compute node and SERVICE_HOST is set to the @@ -112,7 +144,8 @@ Detailed DevStack Instructions to configure similar configuration options for the projects providing those metrics. -#. For the controller node, copy the provided `local.conf.controller`_ example +#. For the controller node, copy the provided `local.conf.controller`_ + (`local_gnocchi.conf.controller`_ if deploying with gnocchi) example file to the controller node's system at ~/devstack/local.conf. Make sure the HOST_IP value is changed appropriately - i.e., HOST_IP is set to the IP address of the controller node. @@ -142,6 +175,17 @@ Detailed DevStack Instructions to FALSE. For Production environment it is suggested to keep it at the default TRUE value. +#. If you want to use prometheus as a datasource, you need to provide a + Prometheus configuration with the compute nodes set as targets, so + it can consume their node-exporter metrics (if you are deploying watcher + with gnocchi as datasource you can skip this step altogether). Copy the + provided `prometheus.yml`_ example file and set the appropriate hostnames + for all the compute nodes (the example configures 2 of them plus the + controller, but you should add all of them if using more than 2 compute + nodes). Set the value of ``PROMETHEUS_CONFIG_FILE`` to the path of the + file you created in the local.conf file (the sample local.conf file uses + ``$DEST`` as the default value for the prometheus config path). + #. Start stacking from the controller node:: ./devstack/stack.sh @@ -154,6 +198,9 @@ Detailed DevStack Instructions .. _local.conf.controller: https://github.com/openstack/watcher/tree/master/devstack/local.conf.controller .. _local.conf.compute: https://github.com/openstack/watcher/tree/master/devstack/local.conf.compute +.. _local_gnocchi.conf.controller: https://github.com/openstack/watcher/tree/master/devstack/local_gnocchi.conf.controller +.. _local_gnocchi.conf.compute: https://github.com/openstack/watcher/tree/master/devstack/local_gnocchi.conf.compute +.. _prometheus.yml: https://github.com/openstack/watcher/tree/master/devstack/prometheus.yml Multi-Node DevStack Environment ===============================