lib/lrmd/lrmd_client.c

/* */
This source file includes following definitions.
lrmd_list_add
lrmd_list_freeall
lrmd_key_value_add
lrmd_key_value_freeall
lrmd_new_event
lrmd_copy_event
lrmd_free_event
lrmd_dispatch_internal
lrmd_ipc_dispatch
lrmd_free_xml
remote_executor_connected
lrmd_tls_dispatch
lrmd_poll
lrmd_dispatch
lrmd_create_op
lrmd_ipc_connection_destroy
lrmd_tls_connection_destroy
lrmd__remote_send_xml
read_remote_reply
send_remote_message
lrmd_tls_send_recv
lrmd_send_xml
lrmd_send_xml_no_reply
lrmd_api_is_connected
lrmd_send_command
lrmd_api_poke_connection
lrmd__validate_remote_settings
lrmd_handshake
lrmd_ipc_connect
copy_gnutls_datum
clear_gnutls_datum
read_gnutls_key
key_is_cached
key_cache_expired
clear_key_cache
get_cached_key
cache_key
get_remote_key
lrmd__init_remote_key
lrmd_gnutls_global_init
report_async_connection_result
lrmd__tls_client_handshake
add_tls_to_mainloop
lrmd_tcp_connect_cb
lrmd_tls_connect_async
lrmd_tls_connect
lrmd_api_connect
lrmd_api_connect_async
lrmd_ipc_disconnect
lrmd_tls_disconnect
lrmd_api_disconnect
lrmd_api_register_rsc
lrmd_api_unregister_rsc
lrmd_new_rsc_info
lrmd_copy_rsc_info
lrmd_free_rsc_info
lrmd_api_get_rsc_info
lrmd_free_op_info
lrmd_api_get_recurring_ops
lrmd_api_set_callback
lrmd_internal_set_proxy_callback
lrmd_internal_proxy_dispatch
lrmd_internal_proxy_send
stonith_get_metadata
lrmd_api_get_metadata
lrmd_api_get_metadata_params
lrmd_api_exec
lrmd_api_exec_alert
lrmd_api_cancel
list_stonith_agents
lrmd_api_list_agents
does_provider_have_agent
lrmd_api_list_ocf_providers
lrmd_api_list_standards
lrmd__new
lrmd_api_new
lrmd_remote_api_new
lrmd_api_delete
metadata_complete
lrmd__metadata_async
lrmd__set_result
lrmd__reset_result
lrmd__uptime
   1 /*
   2  * Copyright 2012-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <errno.h>
  20 
  21 #include <sys/types.h>
  22 #include <sys/wait.h>
  23 
  24 #include <glib.h>
  25 #include <dirent.h>
  26 
  27 #include <crm/crm.h>
  28 #include <crm/lrmd.h>
  29 #include <crm/lrmd_internal.h>
  30 #include <crm/services.h>
  31 #include <crm/services_internal.h>
  32 #include <crm/common/mainloop.h>
  33 #include <crm/common/ipc_internal.h>
  34 #include <crm/common/remote_internal.h>
  35 #include <crm/msg_xml.h>
  36 
  37 #include <crm/stonith-ng.h>
  38 #include <crm/fencing/internal.h>
  39 
  40 #ifdef HAVE_GNUTLS_GNUTLS_H
  41 #  include <gnutls/gnutls.h>
  42 #endif
  43 
  44 #include <sys/socket.h>
  45 #include <netinet/in.h>
  46 #include <netinet/ip.h>
  47 #include <arpa/inet.h>
  48 #include <netdb.h>
  49 
  50 #define MAX_TLS_RECV_WAIT 10000
  51 
  52 CRM_TRACE_INIT_DATA(lrmd);
  53 
  54 static int lrmd_api_disconnect(lrmd_t * lrmd);
  55 static int lrmd_api_is_connected(lrmd_t * lrmd);
  56 
  57 /* IPC proxy functions */
  58 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  59 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  60 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  61 
  62 #ifdef HAVE_GNUTLS_GNUTLS_H
  63 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
  64 gnutls_psk_client_credentials_t psk_cred_s;
  65 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  66 static int global_remote_msg_id = 0;
  67 static void lrmd_tls_connection_destroy(gpointer userdata);
  68 #endif
  69 
  70 typedef struct lrmd_private_s {
  71     uint64_t type;
  72     char *token;
  73     mainloop_io_t *source;
  74 
  75     /* IPC parameters */
  76     crm_ipc_t *ipc;
  77 
  78     pcmk__remote_t *remote;
  79 
  80     /* Extra TLS parameters */
  81     char *remote_nodename;
  82 #ifdef HAVE_GNUTLS_GNUTLS_H
  83     char *server;
  84     int port;
  85     gnutls_psk_client_credentials_t psk_cred_c;
  86 
  87     /* while the async connection is occurring, this is the id
  88      * of the connection timeout timer. */
  89     int async_timer;
  90     int sock;
  91     /* since tls requires a round trip across the network for a
  92      * request/reply, there are times where we just want to be able
  93      * to send a request from the client and not wait around (or even care
  94      * about) what the reply is. */
  95     int expected_late_replies;
  96     GList *pending_notify;
  97     crm_trigger_t *process_notify;
  98 #endif
  99 
 100     lrmd_event_callback callback;
 101 
 102     /* Internal IPC proxy msg passing for remote guests */
 103     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 104     void *proxy_callback_userdata;
 105     char *peer_version;
 106 } lrmd_private_t;
 107 
 108 static lrmd_list_t *
 109 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 110 {
 111     lrmd_list_t *p, *end;
 112 
 113     p = calloc(1, sizeof(lrmd_list_t));
 114     p->val = strdup(value);
 115 
 116     end = head;
 117     while (end && end->next) {
 118         end = end->next;
 119     }
 120 
 121     if (end) {
 122         end->next = p;
 123     } else {
 124         head = p;
 125     }
 126 
 127     return head;
 128 }
 129 
 130 void
 131 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 132 {
 133     lrmd_list_t *p;
 134 
 135     while (head) {
 136         char *val = (char *)head->val;
 137 
 138         p = head->next;
 139         free(val);
 140         free(head);
 141         head = p;
 142     }
 143 }
 144 
 145 lrmd_key_value_t *
 146 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 147 {
 148     lrmd_key_value_t *p, *end;
 149 
 150     p = calloc(1, sizeof(lrmd_key_value_t));
 151     p->key = strdup(key);
 152     p->value = strdup(value);
 153 
 154     end = head;
 155     while (end && end->next) {
 156         end = end->next;
 157     }
 158 
 159     if (end) {
 160         end->next = p;
 161     } else {
 162         head = p;
 163     }
 164 
 165     return head;
 166 }
 167 
 168 void
 169 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 170 {
 171     lrmd_key_value_t *p;
 172 
 173     while (head) {
 174         p = head->next;
 175         free(head->key);
 176         free(head->value);
 177         free(head);
 178         head = p;
 179     }
 180 }
 181 
 182 /*!
 183  * \brief Create a new lrmd_event_data_t object
 184  *
 185  * \param[in] rsc_id       ID of resource involved in event
 186  * \param[in] task         Action name
 187  * \param[in] interval_ms  Action interval
 188  *
 189  * \return Newly allocated and initialized lrmd_event_data_t
 190  * \note This functions asserts on memory errors, so the return value is
 191  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 192  *       result with lrmd_free_event().
 193  */
 194 lrmd_event_data_t *
 195 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 196 {
 197     lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
 198 
 199     CRM_ASSERT(event != NULL);
 200     pcmk__str_update((char **) &event->rsc_id, rsc_id);
 201     pcmk__str_update((char **) &event->op_type, task);
 202     event->interval_ms = interval_ms;
 203     return event;
 204 }
 205 
 206 lrmd_event_data_t *
 207 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 208 {
 209     lrmd_event_data_t *copy = NULL;
 210 
 211     copy = calloc(1, sizeof(lrmd_event_data_t));
 212 
 213     copy->type = event->type;
 214     pcmk__str_update((char **) &copy->rsc_id, event->rsc_id);
 215     pcmk__str_update((char **) &copy->op_type, event->op_type);
 216     pcmk__str_update((char **) &copy->user_data, event->user_data);
 217     copy->call_id = event->call_id;
 218     copy->timeout = event->timeout;
 219     copy->interval_ms = event->interval_ms;
 220     copy->start_delay = event->start_delay;
 221     copy->rsc_deleted = event->rsc_deleted;
 222     copy->rc = event->rc;
 223     copy->op_status = event->op_status;
 224     pcmk__str_update((char **) &copy->output, event->output);
 225     copy->t_run = event->t_run;
 226     copy->t_rcchange = event->t_rcchange;
 227     copy->exec_time = event->exec_time;
 228     copy->queue_time = event->queue_time;
 229     copy->connection_rc = event->connection_rc;
 230     copy->params = pcmk__str_table_dup(event->params);
 231     pcmk__str_update((char **) &copy->remote_nodename, event->remote_nodename);
 232     pcmk__str_update((char **) &copy->exit_reason, event->exit_reason);
 233 
 234     return copy;
 235 }
 236 
 237 /*!
 238  * \brief Free an executor event
 239  *
 240  * \param[in,out]  Executor event object to free
 241  */
 242 void
 243 lrmd_free_event(lrmd_event_data_t *event)
     /*  */
 244 {
 245     if (event == NULL) {
 246         return;
 247     }
 248     // @TODO Why are these const char *?
 249     free((void *) event->rsc_id);
 250     free((void *) event->op_type);
 251     free((void *) event->user_data);
 252     free((void *) event->remote_nodename);
 253     lrmd__reset_result(event);
 254     if (event->params != NULL) {
 255         g_hash_table_destroy(event->params);
 256     }
 257     free(event);
 258 }
 259 
 260 static void
 261 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 262 {
 263     const char *type;
 264     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
 265     lrmd_private_t *native = lrmd->lrmd_private;
 266     lrmd_event_data_t event = { 0, };
 267 
 268     if (proxy_session != NULL) {
 269         /* this is proxy business */
 270         lrmd_internal_proxy_dispatch(lrmd, msg);
 271         return;
 272     } else if (!native->callback) {
 273         /* no callback set */
 274         crm_trace("notify event received but client has not set callback");
 275         return;
 276     }
 277 
 278     event.remote_nodename = native->remote_nodename;
 279     type = crm_element_value(msg, F_LRMD_OPERATION);
 280     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
 281     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 282 
 283     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 284         event.type = lrmd_event_register;
 285     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 286         event.type = lrmd_event_unregister;
 287     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 288         time_t epoch = 0;
 289 
 290         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
 291         crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
 292         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
 293         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
 294         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
 295         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 296 
 297         crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
 298         event.t_run = (unsigned int) epoch;
 299 
 300         crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
 301         event.t_rcchange = (unsigned int) epoch;
 302 
 303         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
 304         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 305 
 306         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
 307         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
 308         event.type = lrmd_event_exec_complete;
 309 
 310         /* output and exit_reason may be freed by a callback */
 311         event.output = crm_element_value_copy(msg, F_LRMD_RSC_OUTPUT);
 312         lrmd__set_result(&event, event.rc, event.op_status,
 313                          crm_element_value(msg, F_LRMD_RSC_EXIT_REASON));
 314 
 315         event.params = xml2list(msg);
 316     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 317         event.type = lrmd_event_new_client;
 318     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 319         event.type = lrmd_event_poke;
 320     } else {
 321         return;
 322     }
 323 
 324     crm_trace("op %s notify event received", type);
 325     native->callback(&event);
 326 
 327     if (event.params) {
 328         g_hash_table_destroy(event.params);
 329     }
 330     lrmd__reset_result(&event);
 331 }
 332 
 333 // \return Always 0, to indicate that IPC mainloop source should be kept
 334 static int
 335 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 336 {
 337     lrmd_t *lrmd = userdata;
 338     lrmd_private_t *native = lrmd->lrmd_private;
 339 
 340     if (native->callback != NULL) {
 341         xmlNode *msg = string2xml(buffer);
 342 
 343         lrmd_dispatch_internal(lrmd, msg);
 344         free_xml(msg);
 345     }
 346     return 0;
 347 }
 348 
 349 #ifdef HAVE_GNUTLS_GNUTLS_H
 350 static void
 351 lrmd_free_xml(gpointer userdata)
     /*  */
 352 {
 353     free_xml((xmlNode *) userdata);
 354 }
 355 
 356 static bool
 357 remote_executor_connected(lrmd_t * lrmd)
     /*  */
 358 {
 359     lrmd_private_t *native = lrmd->lrmd_private;
 360 
 361     return (native->remote->tls_session != NULL);
 362 }
 363 
 364 /*!
 365  * \internal
 366  * \brief TLS dispatch function (for both trigger and file descriptor sources)
 367  *
 368  * \param[in,out] userdata  API connection
 369  *
 370  * \return Always return a nonnegative value, which as a file descriptor
 371  *         dispatch function means keep the mainloop source, and as a
 372  *         trigger dispatch function, 0 means remove the trigger from the
 373  *         mainloop while 1 means keep it (and job completed)
 374  */
 375 static int
 376 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 377 {
 378     lrmd_t *lrmd = userdata;
 379     lrmd_private_t *native = lrmd->lrmd_private;
 380     xmlNode *xml = NULL;
 381     int rc = pcmk_rc_ok;
 382 
 383     if (!remote_executor_connected(lrmd)) {
 384         crm_trace("TLS dispatch triggered after disconnect");
 385         return 0;
 386     }
 387 
 388     crm_trace("TLS dispatch triggered");
 389 
 390     /* First check if there are any pending notifies to process that came
 391      * while we were waiting for replies earlier. */
 392     if (native->pending_notify) {
 393         GList *iter = NULL;
 394 
 395         crm_trace("Processing pending notifies");
 396         for (iter = native->pending_notify; iter; iter = iter->next) {
 397             lrmd_dispatch_internal(lrmd, iter->data);
 398         }
 399         g_list_free_full(native->pending_notify, lrmd_free_xml);
 400         native->pending_notify = NULL;
 401     }
 402 
 403     /* Next read the current buffer and see if there are any messages to handle. */
 404     switch (pcmk__remote_ready(native->remote, 0)) {
 405         case pcmk_rc_ok:
 406             rc = pcmk__read_remote_message(native->remote, -1);
 407             xml = pcmk__remote_message_xml(native->remote);
 408             break;
 409         case ETIME:
 410             // Nothing to read, check if a full message is already in buffer
 411             xml = pcmk__remote_message_xml(native->remote);
 412             break;
 413         default:
 414             rc = ENOTCONN;
 415             break;
 416     }
 417     while (xml) {
 418         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 419         if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 420             lrmd_dispatch_internal(lrmd, xml);
 421         } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 422             if (native->expected_late_replies > 0) {
 423                 native->expected_late_replies--;
 424             } else {
 425                 int reply_id = 0;
 426                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
 427                 /* if this happens, we want to know about it */
 428                 crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 429             }
 430         }
 431         free_xml(xml);
 432         xml = pcmk__remote_message_xml(native->remote);
 433     }
 434 
 435     if (rc == ENOTCONN) {
 436         crm_info("Lost %s executor connection while reading data",
 437                  (native->remote_nodename? native->remote_nodename : "local"));
 438         lrmd_tls_disconnect(lrmd);
 439         return 0;
 440     }
 441     return 1;
 442 }
 443 #endif
 444 
 445 /* Not used with mainloop */
 446 int
 447 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 448 {
 449     lrmd_private_t *native = lrmd->lrmd_private;
 450 
 451     switch (native->type) {
 452         case pcmk__client_ipc:
 453             return crm_ipc_ready(native->ipc);
 454 
 455 #ifdef HAVE_GNUTLS_GNUTLS_H
 456         case pcmk__client_tls:
 457             if (native->pending_notify) {
 458                 return 1;
 459             } else {
 460                 int rc = pcmk__remote_ready(native->remote, 0);
 461 
 462                 switch (rc) {
 463                     case pcmk_rc_ok:
 464                         return 1;
 465                     case ETIME:
 466                         return 0;
 467                     default:
 468                         return pcmk_rc2legacy(rc);
 469                 }
 470             }
 471 #endif
 472         default:
 473             crm_err("Unsupported executor connection type (bug?): %d",
 474                     native->type);
 475             return -EPROTONOSUPPORT;
 476     }
 477 }
 478 
 479 /* Not used with mainloop */
 480 bool
 481 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 482 {
 483     lrmd_private_t *private = NULL;
 484 
 485     CRM_ASSERT(lrmd != NULL);
 486 
 487     private = lrmd->lrmd_private;
 488     switch (private->type) {
 489         case pcmk__client_ipc:
 490             while (crm_ipc_ready(private->ipc)) {
 491                 if (crm_ipc_read(private->ipc) > 0) {
 492                     const char *msg = crm_ipc_buffer(private->ipc);
 493 
 494                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 495                 }
 496             }
 497             break;
 498 #ifdef HAVE_GNUTLS_GNUTLS_H
 499         case pcmk__client_tls:
 500             lrmd_tls_dispatch(lrmd);
 501             break;
 502 #endif
 503         default:
 504             crm_err("Unsupported executor connection type (bug?): %d",
 505                     private->type);
 506     }
 507 
 508     if (lrmd_api_is_connected(lrmd) == FALSE) {
 509         crm_err("Connection closed");
 510         return FALSE;
 511     }
 512 
 513     return TRUE;
 514 }
 515 
 516 static xmlNode *
 517 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 518                enum lrmd_call_options options)
 519 {
 520     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 521 
 522     CRM_CHECK(op_msg != NULL, return NULL);
 523     CRM_CHECK(token != NULL, return NULL);
 524 
 525     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
 526     crm_xml_add(op_msg, F_TYPE, T_LRMD);
 527     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
 528     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
 529     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
 530     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 531 
 532     if (data != NULL) {
 533         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
 534     }
 535 
 536     crm_trace("Created executor %s command with call options %.8lx (%d)",
 537               op, (long)options, options);
 538     return op_msg;
 539 }
 540 
 541 static void
 542 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 543 {
 544     lrmd_t *lrmd = userdata;
 545     lrmd_private_t *native = lrmd->lrmd_private;
 546 
 547     crm_info("IPC connection destroyed");
 548 
 549     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 550     native->ipc = NULL;
 551     native->source = NULL;
 552 
 553     if (native->callback) {
 554         lrmd_event_data_t event = { 0, };
 555         event.type = lrmd_event_disconnect;
 556         event.remote_nodename = native->remote_nodename;
 557         native->callback(&event);
 558     }
 559 }
 560 
 561 #ifdef HAVE_GNUTLS_GNUTLS_H
 562 static void
 563 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 564 {
 565     lrmd_t *lrmd = userdata;
 566     lrmd_private_t *native = lrmd->lrmd_private;
 567 
 568     crm_info("TLS connection destroyed");
 569 
 570     if (native->remote->tls_session) {
 571         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 572         gnutls_deinit(*native->remote->tls_session);
 573         gnutls_free(native->remote->tls_session);
 574     }
 575     if (native->psk_cred_c) {
 576         gnutls_psk_free_client_credentials(native->psk_cred_c);
 577     }
 578     if (native->sock) {
 579         close(native->sock);
 580     }
 581     if (native->process_notify) {
 582         mainloop_destroy_trigger(native->process_notify);
 583         native->process_notify = NULL;
 584     }
 585     if (native->pending_notify) {
 586         g_list_free_full(native->pending_notify, lrmd_free_xml);
 587         native->pending_notify = NULL;
 588     }
 589 
 590     free(native->remote->buffer);
 591     native->remote->buffer = NULL;
 592     native->source = 0;
 593     native->sock = 0;
 594     native->psk_cred_c = NULL;
 595     native->remote->tls_session = NULL;
 596     native->sock = 0;
 597 
 598     if (native->callback) {
 599         lrmd_event_data_t event = { 0, };
 600         event.remote_nodename = native->remote_nodename;
 601         event.type = lrmd_event_disconnect;
 602         native->callback(&event);
 603     }
 604     return;
 605 }
 606 
 607 // \return Standard Pacemaker return code
 608 int
 609 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 610                       const char *msg_type)
 611 {
 612     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
 613     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
 614     return pcmk__remote_send_xml(session, msg);
 615 }
 616 
 617 // \return Standard Pacemaker return code
 618 static int
 619 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
     /*  */
 620                   xmlNode **reply)
 621 {
 622     lrmd_private_t *native = lrmd->lrmd_private;
 623     time_t start = time(NULL);
 624     const char *msg_type = NULL;
 625     int reply_id = 0;
 626     int remaining_timeout = 0;
 627     int rc = pcmk_rc_ok;
 628 
 629     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 630      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 631     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 632         total_timeout = MAX_TLS_RECV_WAIT;
 633     }
 634 
 635     for (*reply = NULL; *reply == NULL; ) {
 636 
 637         *reply = pcmk__remote_message_xml(native->remote);
 638         if (*reply == NULL) {
 639             /* read some more off the tls buffer if we still have time left. */
 640             if (remaining_timeout) {
 641                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 642             } else {
 643                 remaining_timeout = total_timeout;
 644             }
 645             if (remaining_timeout <= 0) {
 646                 return ETIME;
 647             }
 648 
 649             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
 650             if (rc != pcmk_rc_ok) {
 651                 return rc;
 652             }
 653 
 654             *reply = pcmk__remote_message_xml(native->remote);
 655             if (*reply == NULL) {
 656                 return ENOMSG;
 657             }
 658         }
 659 
 660         crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
 661         msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);
 662 
 663         if (!msg_type) {
 664             crm_err("Empty msg type received while waiting for reply");
 665             free_xml(*reply);
 666             *reply = NULL;
 667         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 668             /* got a notify while waiting for reply, trigger the notify to be processed later */
 669             crm_info("queueing notify");
 670             native->pending_notify = g_list_append(native->pending_notify, *reply);
 671             if (native->process_notify) {
 672                 crm_info("notify trigger set.");
 673                 mainloop_set_trigger(native->process_notify);
 674             }
 675             *reply = NULL;
 676         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 677             /* msg isn't a reply, make some noise */
 678             crm_err("Expected a reply, got %s", msg_type);
 679             free_xml(*reply);
 680             *reply = NULL;
 681         } else if (reply_id != expected_reply_id) {
 682             if (native->expected_late_replies > 0) {
 683                 native->expected_late_replies--;
 684             } else {
 685                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 686             }
 687             free_xml(*reply);
 688             *reply = NULL;
 689         }
 690     }
 691 
 692     if (native->remote->buffer && native->process_notify) {
 693         mainloop_set_trigger(native->process_notify);
 694     }
 695 
 696     return rc;
 697 }
 698 
 699 // \return Standard Pacemaker return code
 700 static int
 701 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
     /*  */
 702 {
 703     int rc = pcmk_rc_ok;
 704     lrmd_private_t *native = lrmd->lrmd_private;
 705 
 706     global_remote_msg_id++;
 707     if (global_remote_msg_id <= 0) {
 708         global_remote_msg_id = 1;
 709     }
 710 
 711     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
 712                                "request");
 713     if (rc != pcmk_rc_ok) {
 714         crm_err("Disconnecting because TLS message could not be sent to "
 715                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 716         lrmd_tls_disconnect(lrmd);
 717     }
 718     return rc;
 719 }
 720 
 721 static int
 722 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 723 {
 724     int rc = 0;
 725     xmlNode *xml = NULL;
 726 
 727     if (!remote_executor_connected(lrmd)) {
 728         return -ENOTCONN;
 729     }
 730 
 731     rc = send_remote_message(lrmd, msg);
 732     if (rc != pcmk_rc_ok) {
 733         return pcmk_rc2legacy(rc);
 734     }
 735 
 736     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
 737     if (rc != pcmk_rc_ok) {
 738         crm_err("Disconnecting remote after request %d reply not received: %s "
 739                 CRM_XS " rc=%d timeout=%dms",
 740                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
 741         lrmd_tls_disconnect(lrmd);
 742     }
 743 
 744     if (reply) {
 745         *reply = xml;
 746     } else {
 747         free_xml(xml);
 748     }
 749 
 750     return pcmk_rc2legacy(rc);
 751 }
 752 #endif
 753 
 754 static int
 755 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 756 {
 757     int rc = pcmk_ok;
 758     lrmd_private_t *native = lrmd->lrmd_private;
 759 
 760     switch (native->type) {
 761         case pcmk__client_ipc:
 762             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 763             break;
 764 #ifdef HAVE_GNUTLS_GNUTLS_H
 765         case pcmk__client_tls:
 766             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 767             break;
 768 #endif
 769         default:
 770             crm_err("Unsupported executor connection type (bug?): %d",
 771                     native->type);
 772             rc = -EPROTONOSUPPORT;
 773     }
 774 
 775     return rc;
 776 }
 777 
 778 static int
 779 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 780 {
 781     int rc = pcmk_ok;
 782     lrmd_private_t *native = lrmd->lrmd_private;
 783 
 784     switch (native->type) {
 785         case pcmk__client_ipc:
 786             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 787             break;
 788 #ifdef HAVE_GNUTLS_GNUTLS_H
 789         case pcmk__client_tls:
 790             rc = send_remote_message(lrmd, msg);
 791             if (rc == pcmk_rc_ok) {
 792                 /* we don't want to wait around for the reply, but
 793                  * since the request/reply protocol needs to behave the same
 794                  * as libqb, a reply will eventually come later anyway. */
 795                 native->expected_late_replies++;
 796             }
 797             rc = pcmk_rc2legacy(rc);
 798             break;
 799 #endif
 800         default:
 801             crm_err("Unsupported executor connection type (bug?): %d",
 802                     native->type);
 803             rc = -EPROTONOSUPPORT;
 804     }
 805 
 806     return rc;
 807 }
 808 
 809 static int
 810 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 811 {
 812     lrmd_private_t *native = lrmd->lrmd_private;
 813 
 814     switch (native->type) {
 815         case pcmk__client_ipc:
 816             return crm_ipc_connected(native->ipc);
 817 #ifdef HAVE_GNUTLS_GNUTLS_H
 818         case pcmk__client_tls:
 819             return remote_executor_connected(lrmd);
 820 #endif
 821         default:
 822             crm_err("Unsupported executor connection type (bug?): %d",
 823                     native->type);
 824             return 0;
 825     }
 826 }
 827 
 828 /*!
 829  * \internal
 830  * \brief Send a prepared API command to the executor
 831  *
 832  * \param[in,out] lrmd          Existing connection to the executor
 833  * \param[in]     op            Name of API command to send
 834  * \param[in]     data          Command data XML to add to the sent command
 835  * \param[out]    output_data   If expecting a reply, it will be stored here
 836  * \param[in]     timeout       Timeout in milliseconds (if 0, defaults to
 837  *                              a sensible value per the type of connection,
 838  *                              standard vs. pacemaker remote);
 839  *                              also propagated to the command XML
 840  * \param[in]     call_options  Call options to pass to server when sending
 841  * \param[in]     expect_reply  If TRUE, wait for a reply from the server;
 842  *                              must be TRUE for IPC (as opposed to TLS) clients
 843  *
 844  * \return pcmk_ok on success, -errno on error
 845  */
 846 static int
 847 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 848                   xmlNode **output_data, int timeout,
 849                   enum lrmd_call_options options, gboolean expect_reply)
 850 {
 851     int rc = pcmk_ok;
 852     lrmd_private_t *native = lrmd->lrmd_private;
 853     xmlNode *op_msg = NULL;
 854     xmlNode *op_reply = NULL;
 855 
 856     if (!lrmd_api_is_connected(lrmd)) {
 857         return -ENOTCONN;
 858     }
 859 
 860     if (op == NULL) {
 861         crm_err("No operation specified");
 862         return -EINVAL;
 863     }
 864 
 865     CRM_CHECK(native->token != NULL,;
 866         );
 867     crm_trace("Sending %s op to executor", op);
 868 
 869     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 870 
 871     if (op_msg == NULL) {
 872         return -EINVAL;
 873     }
 874 
 875     if (expect_reply) {
 876         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 877     } else {
 878         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 879         goto done;
 880     }
 881 
 882     if (rc < 0) {
 883         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 884         goto done;
 885 
 886     } else if(op_reply == NULL) {
 887         rc = -ENOMSG;
 888         goto done;
 889     }
 890 
 891     rc = pcmk_ok;
 892     crm_trace("%s op reply received", op);
 893     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
 894         rc = -ENOMSG;
 895         goto done;
 896     }
 897 
 898     crm_log_xml_trace(op_reply, "Reply");
 899 
 900     if (output_data) {
 901         *output_data = op_reply;
 902         op_reply = NULL;        /* Prevent subsequent free */
 903     }
 904 
 905   done:
 906     if (lrmd_api_is_connected(lrmd) == FALSE) {
 907         crm_err("Executor disconnected");
 908     }
 909 
 910     free_xml(op_msg);
 911     free_xml(op_reply);
 912     return rc;
 913 }
 914 
 915 static int
 916 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 917 {
 918     int rc;
 919     lrmd_private_t *native = lrmd->lrmd_private;
 920     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 921 
 922     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 923     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 924                            (native->type == pcmk__client_ipc));
 925     free_xml(data);
 926 
 927     return rc < 0 ? rc : pcmk_ok;
 928 }
 929 
 930 // \return Standard Pacemaker return code
 931 int
 932 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
     /*  */
 933 {
 934     int rc = pcmk_rc_ok;
 935     const char *value;
 936     lrmd_private_t *native = lrmd->lrmd_private;
 937     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 938 
 939     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 940 
 941     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
 942     if ((value) &&
 943         (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
 944        crm_xml_add(data, F_LRMD_WATCHDOG, value);
 945     }
 946 
 947     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
 948                            (native->type == pcmk__client_ipc));
 949     free_xml(data);
 950     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
 951 }
 952 
 953 static int
 954 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
 955 {
 956     int rc = pcmk_ok;
 957     lrmd_private_t *native = lrmd->lrmd_private;
 958     xmlNode *reply = NULL;
 959     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 960 
 961     crm_xml_add(hello, F_TYPE, T_LRMD);
 962     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
 963     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
 964     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 965 
 966     /* advertise that we are a proxy provider */
 967     if (native->proxy_callback) {
 968         pcmk__xe_set_bool_attr(hello, F_LRMD_IS_IPC_PROVIDER, true);
 969     }
 970 
 971     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 972 
 973     if (rc < 0) {
 974         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
 975         rc = -ECOMM;
 976     } else if (reply == NULL) {
 977         crm_err("Did not receive registration reply");
 978         rc = -EPROTO;
 979     } else {
 980         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
 981         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
 982         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 983         long long uptime = -1;
 984 
 985         crm_element_value_int(reply, F_LRMD_RC, &rc);
 986 
 987         /* The remote executor may add its uptime to the XML reply, which is
 988          * useful in handling transient attributes when the connection to the
 989          * remote node unexpectedly drops.  If no parameter is given, just
 990          * default to -1.
 991          */
 992         crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime);
 993         native->remote->uptime = uptime;
 994 
 995         if (rc == -EPROTO) {
 996             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
 997                 LRMD_PROTOCOL_VERSION, version);
 998             crm_log_xml_err(reply, "Protocol Error");
 999 
1000         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
1001             crm_err("Invalid registration message: %s", msg_type);
1002             crm_log_xml_err(reply, "Bad reply");
1003             rc = -EPROTO;
1004         } else if (tmp_ticket == NULL) {
1005             crm_err("No registration token provided");
1006             crm_log_xml_err(reply, "Bad reply");
1007             rc = -EPROTO;
1008         } else {
1009             crm_trace("Obtained registration token: %s", tmp_ticket);
1010             native->token = strdup(tmp_ticket);
1011             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
1012             rc = pcmk_ok;
1013         }
1014     }
1015 
1016     free_xml(reply);
1017     free_xml(hello);
1018 
1019     if (rc != pcmk_ok) {
1020         lrmd_api_disconnect(lrmd);
1021     }
1022     return rc;
1023 }
1024 
1025 static int
1026 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1027 {
1028     int rc = pcmk_ok;
1029     lrmd_private_t *native = lrmd->lrmd_private;
1030 
1031     struct ipc_client_callbacks lrmd_callbacks = {
1032         .dispatch = lrmd_ipc_dispatch,
1033         .destroy = lrmd_ipc_connection_destroy
1034     };
1035 
1036     crm_info("Connecting to executor");
1037 
1038     if (fd) {
1039         /* No mainloop */
1040         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1041         if (native->ipc && crm_ipc_connect(native->ipc)) {
1042             *fd = crm_ipc_get_fd(native->ipc);
1043         } else if (native->ipc) {
1044             crm_perror(LOG_ERR, "Connection to executor failed");
1045             rc = -ENOTCONN;
1046         }
1047     } else {
1048         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1049         native->ipc = mainloop_get_ipc_client(native->source);
1050     }
1051 
1052     if (native->ipc == NULL) {
1053         crm_debug("Could not connect to the executor API");
1054         rc = -ENOTCONN;
1055     }
1056 
1057     return rc;
1058 }
1059 
1060 #ifdef HAVE_GNUTLS_GNUTLS_H
1061 static void
1062 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1063 {
1064     CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));
1065 
1066     dest->data = gnutls_malloc(source->size);
1067     CRM_ASSERT(dest->data);
1068 
1069     memcpy(dest->data, source->data, source->size);
1070     dest->size = source->size;
1071 }
1072 
1073 static void
1074 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1075 {
1076     gnutls_free(datum->data);
1077     datum->data = NULL;
1078     datum->size = 0;
1079 }
1080 
1081 #define KEY_READ_LEN 256    // Chunk size for reading key from file
1082 
1083 // \return Standard Pacemaker return code
1084 static int
1085 read_gnutls_key(const char *location, gnutls_datum_t *key)
     /*  */
1086 {
1087     FILE *stream = NULL;
1088     size_t buf_len = KEY_READ_LEN;
1089 
1090     if ((location == NULL) || (key == NULL)) {
1091         return EINVAL;
1092     }
1093 
1094     stream = fopen(location, "r");
1095     if (stream == NULL) {
1096         return errno;
1097     }
1098 
1099     key->data = gnutls_malloc(buf_len);
1100     key->size = 0;
1101     while (!feof(stream)) {
1102         int next = fgetc(stream);
1103 
1104         if (next == EOF) {
1105             if (!feof(stream)) {
1106                 crm_warn("Pacemaker Remote key read was partially successful "
1107                          "(copy in memory may be corrupted)");
1108             }
1109             break;
1110         }
1111         if (key->size == buf_len) {
1112             buf_len = key->size + KEY_READ_LEN;
1113             key->data = gnutls_realloc(key->data, buf_len);
1114             CRM_ASSERT(key->data);
1115         }
1116         key->data[key->size++] = (unsigned char) next;
1117     }
1118     fclose(stream);
1119 
1120     if (key->size == 0) {
1121         clear_gnutls_datum(key);
1122         return ENOKEY;
1123     }
1124     return pcmk_rc_ok;
1125 }
1126 
1127 // Cache the most recently used Pacemaker Remote authentication key
1128 
1129 struct key_cache_s {
1130     time_t updated;         // When cached key was read (valid for 1 minute)
1131     const char *location;   // Where cached key was read from
1132     gnutls_datum_t key;     // Cached key
1133 };
1134 
1135 static bool
1136 key_is_cached(struct key_cache_s *key_cache)
     /*  */
1137 {
1138     return key_cache->updated != 0;
1139 }
1140 
1141 static bool
1142 key_cache_expired(struct key_cache_s *key_cache)
     /*  */
1143 {
1144     return (time(NULL) - key_cache->updated) >= 60;
1145 }
1146 
1147 static void
1148 clear_key_cache(struct key_cache_s *key_cache)
     /*  */
1149 {
1150     clear_gnutls_datum(&(key_cache->key));
1151     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
1152         key_cache->updated = 0;
1153         key_cache->location = NULL;
1154         crm_debug("Cleared Pacemaker Remote key cache");
1155     }
1156 }
1157 
1158 static void
1159 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
     /*  */
1160 {
1161     copy_gnutls_datum(key, &(key_cache->key));
1162     crm_debug("Using cached Pacemaker Remote key from %s",
1163               pcmk__s(key_cache->location, "unknown location"));
1164 }
1165 
1166 static void
1167 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
     /*  */
1168           const char *location)
1169 {
1170     key_cache->updated = time(NULL);
1171     key_cache->location = location;
1172     copy_gnutls_datum(&(key_cache->key), key);
1173     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
1174               pcmk__s(location, "unknown location"));
1175 }
1176 
1177 /*!
1178  * \internal
1179  * \brief Get Pacemaker Remote authentication key from file or cache
1180  *
1181  * \param[in]  location         Path to key file to try (this memory must
1182  *                              persist across all calls of this function)
1183  * \param[out] key              Key from location or cache
1184  *
1185  * \return Standard Pacemaker return code
1186  */
1187 static int
1188 get_remote_key(const char *location, gnutls_datum_t *key)
     /*  */
1189 {
1190     static struct key_cache_s key_cache = { 0, };
1191     int rc = pcmk_rc_ok;
1192 
1193     if ((location == NULL) || (key == NULL)) {
1194         return EINVAL;
1195     }
1196 
1197     if (key_is_cached(&key_cache)) {
1198         if (key_cache_expired(&key_cache)) {
1199             clear_key_cache(&key_cache);
1200         } else {
1201             get_cached_key(&key_cache, key);
1202             return pcmk_rc_ok;
1203         }
1204     }
1205 
1206     rc = read_gnutls_key(location, key);
1207     if (rc != pcmk_rc_ok) {
1208         return rc;
1209     }
1210     cache_key(&key_cache, key, location);
1211     return pcmk_rc_ok;
1212 }
1213 
1214 /*!
1215  * \internal
1216  * \brief Initialize the Pacemaker Remote authentication key
1217  *
1218  * Try loading the Pacemaker Remote authentication key from cache if available,
1219  * otherwise from these locations, in order of preference: the value of the
1220  * PCMK_authkey_location environment variable, if set; the Pacemaker default key
1221  * file location; or (for historical reasons) /etc/corosync/authkey.
1222  *
1223  * \param[out] key  Where to store key
1224  *
1225  * \return Standard Pacemaker return code
1226  */
1227 int
1228 lrmd__init_remote_key(gnutls_datum_t *key)
     /*  */
1229 {
1230     static const char *env_location = NULL;
1231     static bool need_env = true;
1232 
1233     int env_rc = pcmk_rc_ok;
1234     int default_rc = pcmk_rc_ok;
1235     int alt_rc = pcmk_rc_ok;
1236 
1237     bool env_is_default = false;
1238     bool env_is_fallback = false;
1239 
1240     if (need_env) {
1241         env_location = getenv("PCMK_authkey_location");
1242         need_env = false;
1243     }
1244 
1245     // Try location in environment variable, if set
1246     if (env_location != NULL) {
1247         env_rc = get_remote_key(env_location, key);
1248         if (env_rc == pcmk_rc_ok) {
1249             return pcmk_rc_ok;
1250         }
1251 
1252         env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
1253         env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
1254 
1255         /* @TODO It would be more secure to fail, rather than fall back to the
1256          * default, if an explicitly set key location is not readable, and it
1257          * would be better to never use the Corosync location as a fallback.
1258          * However, that would break any deployments currently working with the
1259          * fallbacks.
1260          */
1261     }
1262 
1263     // Try default location, if environment wasn't explicitly set to it
1264     if (env_is_default) {
1265         default_rc = env_rc;
1266     } else {
1267         default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
1268     }
1269 
1270     // Try fallback location, if environment wasn't set to it and default failed
1271     if (env_is_fallback) {
1272         alt_rc = env_rc;
1273     } else if (default_rc != pcmk_rc_ok) {
1274         alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
1275     }
1276 
1277     // We have all results, so log and return
1278 
1279     if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
1280         && (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
1281 
1282         crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
1283                  env_location,
1284                  env_is_default? "" : "or default location ",
1285                  env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
1286                  !env_is_default && !env_is_fallback? " " : "",
1287                  env_is_fallback? "" : "or fallback location ",
1288                  env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
1289                  pcmk_rc_str(env_rc));
1290         return ENOKEY;
1291     }
1292 
1293     if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
1294         crm_warn("Could not read Pacemaker Remote key from %s "
1295                  "(using %s location %s instead): %s",
1296                  env_location,
1297                  (default_rc == pcmk_rc_ok)? "default" : "fallback",
1298                  (default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
1299                  pcmk_rc_str(env_rc));
1300         return pcmk_rc_ok;
1301     }
1302 
1303     if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
1304         // Environment unset, defaults failed
1305         crm_warn("Could not read Pacemaker Remote key from default location %s"
1306                  " (or fallback location %s): %s",
1307                  DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
1308                  pcmk_rc_str(default_rc));
1309         return ENOKEY;
1310     }
1311 
1312     return pcmk_rc_ok; // Environment variable unset, a default worked
1313 }
1314 
1315 static void
1316 lrmd_gnutls_global_init(void)
     /*  */
1317 {
1318     static int gnutls_init = 0;
1319 
1320     if (!gnutls_init) {
1321         crm_gnutls_global_init();
1322     }
1323     gnutls_init = 1;
1324 }
1325 #endif
1326 
1327 static void
1328 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1329 {
1330     lrmd_private_t *native = lrmd->lrmd_private;
1331 
1332     if (native->callback) {
1333         lrmd_event_data_t event = { 0, };
1334         event.type = lrmd_event_connect;
1335         event.remote_nodename = native->remote_nodename;
1336         event.connection_rc = rc;
1337         native->callback(&event);
1338     }
1339 }
1340 
1341 #ifdef HAVE_GNUTLS_GNUTLS_H
1342 static inline int
1343 lrmd__tls_client_handshake(pcmk__remote_t *remote)
     /*  */
1344 {
1345     return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
1346 }
1347 
1348 /*!
1349  * \internal
1350  * \brief Add trigger and file descriptor mainloop sources for TLS
1351  *
1352  * \param[in,out] lrmd          API connection with established TLS session
1353  * \param[in]     do_handshake  Whether to perform executor handshake
1354  *
1355  * \return Standard Pacemaker return code
1356  */
1357 static int
1358 add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
     /*  */
1359 {
1360     lrmd_private_t *native = lrmd->lrmd_private;
1361     int rc = pcmk_rc_ok;
1362 
1363     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1364                                    native->server, native->port);
1365 
1366     struct mainloop_fd_callbacks tls_fd_callbacks = {
1367         .dispatch = lrmd_tls_dispatch,
1368         .destroy = lrmd_tls_connection_destroy,
1369     };
1370 
1371     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1372                                                   lrmd_tls_dispatch, lrmd);
1373     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1374                                      &tls_fd_callbacks);
1375 
1376     /* Async connections lose the client name provided by the API caller, so we
1377      * have to use our generated name here to perform the executor handshake.
1378      *
1379      * @TODO Keep track of the caller-provided name. Perhaps we should be using
1380      * that name in this function instead of generating one anyway.
1381      */
1382     if (do_handshake) {
1383         rc = lrmd_handshake(lrmd, name);
1384         rc = pcmk_legacy2rc(rc);
1385     }
1386     free(name);
1387     return rc;
1388 }
1389 
1390 static void
1391 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1392 {
1393     lrmd_t *lrmd = userdata;
1394     lrmd_private_t *native = lrmd->lrmd_private;
1395     gnutls_datum_t psk_key = { NULL, 0 };
1396 
1397     native->async_timer = 0;
1398 
1399     if (rc != pcmk_rc_ok) {
1400         lrmd_tls_connection_destroy(lrmd);
1401         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1402                  CRM_XS " rc=%d",
1403                  native->server, native->port, pcmk_rc_str(rc), rc);
1404         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1405         return;
1406     }
1407 
1408     /* The TCP connection was successful, so establish the TLS connection.
1409      * @TODO make this async to avoid blocking code in client
1410      */
1411 
1412     native->sock = sock;
1413 
1414     rc = lrmd__init_remote_key(&psk_key);
1415     if (rc != pcmk_rc_ok) {
1416         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1417                  CRM_XS " rc=%d",
1418                  native->server, native->port, pcmk_rc_str(rc), rc);
1419         lrmd_tls_connection_destroy(lrmd);
1420         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1421         return;
1422     }
1423 
1424     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1425     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1426     gnutls_free(psk_key.data);
1427 
1428     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
1429                                                         GNUTLS_CRD_PSK,
1430                                                         native->psk_cred_c);
1431     if (native->remote->tls_session == NULL) {
1432         lrmd_tls_connection_destroy(lrmd);
1433         report_async_connection_result(lrmd, -EPROTO);
1434         return;
1435     }
1436 
1437     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1438         crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
1439                  native->server, native->port);
1440         gnutls_deinit(*native->remote->tls_session);
1441         gnutls_free(native->remote->tls_session);
1442         native->remote->tls_session = NULL;
1443         lrmd_tls_connection_destroy(lrmd);
1444         report_async_connection_result(lrmd, -EKEYREJECTED);
1445         return;
1446     }
1447 
1448     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1449              native->server, native->port);
1450     rc = add_tls_to_mainloop(lrmd, true);
1451     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1452 }
1453 
1454 static int
1455 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1456 {
1457     int rc;
1458     int timer_id = 0;
1459     lrmd_private_t *native = lrmd->lrmd_private;
1460 
1461     lrmd_gnutls_global_init();
1462     native->sock = -1;
1463     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1464                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1465     if (rc != pcmk_rc_ok) {
1466         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1467                  CRM_XS " rc=%d",
1468                  native->server, native->port, pcmk_rc_str(rc), rc);
1469         return pcmk_rc2legacy(rc);
1470     }
1471     native->async_timer = timer_id;
1472     return pcmk_ok;
1473 }
1474 
1475 static int
1476 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1477 {
1478     int rc;
1479 
1480     lrmd_private_t *native = lrmd->lrmd_private;
1481     gnutls_datum_t psk_key = { NULL, 0 };
1482 
1483     lrmd_gnutls_global_init();
1484 
1485     native->sock = -1;
1486     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1487                               &(native->sock), NULL, NULL);
1488     if (rc != pcmk_rc_ok) {
1489         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1490                  CRM_XS " rc=%d",
1491                  native->server, native->port, pcmk_rc_str(rc), rc);
1492         lrmd_tls_connection_destroy(lrmd);
1493         return -ENOTCONN;
1494     }
1495 
1496     rc = lrmd__init_remote_key(&psk_key);
1497     if (rc != pcmk_rc_ok) {
1498         lrmd_tls_connection_destroy(lrmd);
1499         return pcmk_rc2legacy(rc);
1500     }
1501 
1502     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1503     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1504     gnutls_free(psk_key.data);
1505 
1506     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
1507                                                         GNUTLS_CRD_PSK,
1508                                                         native->psk_cred_c);
1509     if (native->remote->tls_session == NULL) {
1510         lrmd_tls_connection_destroy(lrmd);
1511         return -EPROTO;
1512     }
1513 
1514     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1515         crm_err("Session creation for %s:%d failed", native->server, native->port);
1516         gnutls_deinit(*native->remote->tls_session);
1517         gnutls_free(native->remote->tls_session);
1518         native->remote->tls_session = NULL;
1519         lrmd_tls_connection_destroy(lrmd);
1520         return -EKEYREJECTED;
1521     }
1522 
1523     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1524              native->port);
1525 
1526     if (fd) {
1527         *fd = native->sock;
1528     } else {
1529         add_tls_to_mainloop(lrmd, false);
1530     }
1531     return pcmk_ok;
1532 }
1533 #endif
1534 
1535 static int
1536 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1537 {
1538     int rc = -ENOTCONN;
1539     lrmd_private_t *native = lrmd->lrmd_private;
1540 
1541     switch (native->type) {
1542         case pcmk__client_ipc:
1543             rc = lrmd_ipc_connect(lrmd, fd);
1544             break;
1545 #ifdef HAVE_GNUTLS_GNUTLS_H
1546         case pcmk__client_tls:
1547             rc = lrmd_tls_connect(lrmd, fd);
1548             break;
1549 #endif
1550         default:
1551             crm_err("Unsupported executor connection type (bug?): %d",
1552                     native->type);
1553             rc = -EPROTONOSUPPORT;
1554     }
1555 
1556     if (rc == pcmk_ok) {
1557         rc = lrmd_handshake(lrmd, name);
1558     }
1559 
1560     return rc;
1561 }
1562 
1563 static int
1564 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1565 {
1566     int rc = pcmk_ok;
1567     lrmd_private_t *native = lrmd->lrmd_private;
1568 
1569     CRM_CHECK(native && native->callback, return -EINVAL);
1570 
1571     switch (native->type) {
1572         case pcmk__client_ipc:
1573             /* fake async connection with ipc.  it should be fast
1574              * enough that we gain very little from async */
1575             rc = lrmd_api_connect(lrmd, name, NULL);
1576             if (!rc) {
1577                 report_async_connection_result(lrmd, rc);
1578             }
1579             break;
1580 #ifdef HAVE_GNUTLS_GNUTLS_H
1581         case pcmk__client_tls:
1582             rc = lrmd_tls_connect_async(lrmd, timeout);
1583             if (rc) {
1584                 /* connection failed, report rc now */
1585                 report_async_connection_result(lrmd, rc);
1586             }
1587             break;
1588 #endif
1589         default:
1590             crm_err("Unsupported executor connection type (bug?): %d",
1591                     native->type);
1592             rc = -EPROTONOSUPPORT;
1593     }
1594 
1595     return rc;
1596 }
1597 
1598 static void
1599 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1600 {
1601     lrmd_private_t *native = lrmd->lrmd_private;
1602 
1603     if (native->source != NULL) {
1604         /* Attached to mainloop */
1605         mainloop_del_ipc_client(native->source);
1606         native->source = NULL;
1607         native->ipc = NULL;
1608 
1609     } else if (native->ipc) {
1610         /* Not attached to mainloop */
1611         crm_ipc_t *ipc = native->ipc;
1612 
1613         native->ipc = NULL;
1614         crm_ipc_close(ipc);
1615         crm_ipc_destroy(ipc);
1616     }
1617 }
1618 
1619 #ifdef HAVE_GNUTLS_GNUTLS_H
1620 static void
1621 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1622 {
1623     lrmd_private_t *native = lrmd->lrmd_private;
1624 
1625     if (native->remote->tls_session) {
1626         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1627         gnutls_deinit(*native->remote->tls_session);
1628         gnutls_free(native->remote->tls_session);
1629         native->remote->tls_session = 0;
1630     }
1631 
1632     if (native->async_timer) {
1633         g_source_remove(native->async_timer);
1634         native->async_timer = 0;
1635     }
1636 
1637     if (native->source != NULL) {
1638         /* Attached to mainloop */
1639         mainloop_del_ipc_client(native->source);
1640         native->source = NULL;
1641 
1642     } else if (native->sock) {
1643         close(native->sock);
1644         native->sock = 0;
1645     }
1646 
1647     if (native->pending_notify) {
1648         g_list_free_full(native->pending_notify, lrmd_free_xml);
1649         native->pending_notify = NULL;
1650     }
1651 }
1652 #endif
1653 
1654 static int
1655 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1656 {
1657     lrmd_private_t *native = lrmd->lrmd_private;
1658     int rc = pcmk_ok;
1659 
1660     crm_info("Disconnecting %s %s executor connection",
1661              pcmk__client_type_str(native->type),
1662              (native->remote_nodename? native->remote_nodename : "local"));
1663     switch (native->type) {
1664         case pcmk__client_ipc:
1665             lrmd_ipc_disconnect(lrmd);
1666             break;
1667 #ifdef HAVE_GNUTLS_GNUTLS_H
1668         case pcmk__client_tls:
1669             lrmd_tls_disconnect(lrmd);
1670             break;
1671 #endif
1672         default:
1673             crm_err("Unsupported executor connection type (bug?): %d",
1674                     native->type);
1675             rc = -EPROTONOSUPPORT;
1676     }
1677 
1678     free(native->token);
1679     native->token = NULL;
1680 
1681     free(native->peer_version);
1682     native->peer_version = NULL;
1683     return rc;
1684 }
1685 
1686 static int
1687 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1688                       const char *rsc_id,
1689                       const char *class,
1690                       const char *provider, const char *type, enum lrmd_call_options options)
1691 {
1692     int rc = pcmk_ok;
1693     xmlNode *data = NULL;
1694 
1695     if (!class || !type || !rsc_id) {
1696         return -EINVAL;
1697     }
1698     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1699         && (provider == NULL)) {
1700         return -EINVAL;
1701     }
1702 
1703     data = create_xml_node(NULL, F_LRMD_RSC);
1704 
1705     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1706     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1707     crm_xml_add(data, F_LRMD_CLASS, class);
1708     crm_xml_add(data, F_LRMD_PROVIDER, provider);
1709     crm_xml_add(data, F_LRMD_TYPE, type);
1710     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1711     free_xml(data);
1712 
1713     return rc;
1714 }
1715 
1716 static int
1717 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1718 {
1719     int rc = pcmk_ok;
1720     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1721 
1722     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1723     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1724     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1725     free_xml(data);
1726 
1727     return rc;
1728 }
1729 
1730 lrmd_rsc_info_t *
1731 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1732                   const char *provider, const char *type)
1733 {
1734     lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
1735 
1736     CRM_ASSERT(rsc_info);
1737     pcmk__str_update(&rsc_info->id, rsc_id);
1738     pcmk__str_update(&rsc_info->standard, standard);
1739     pcmk__str_update(&rsc_info->provider, provider);
1740     pcmk__str_update(&rsc_info->type, type);
1741     return rsc_info;
1742 }
1743 
1744 lrmd_rsc_info_t *
1745 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1746 {
1747     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1748                              rsc_info->provider, rsc_info->type);
1749 }
1750 
1751 void
1752 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1753 {
1754     if (!rsc_info) {
1755         return;
1756     }
1757     free(rsc_info->id);
1758     free(rsc_info->type);
1759     free(rsc_info->standard);
1760     free(rsc_info->provider);
1761     free(rsc_info);
1762 }
1763 
1764 static lrmd_rsc_info_t *
1765 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1766 {
1767     lrmd_rsc_info_t *rsc_info = NULL;
1768     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1769     xmlNode *output = NULL;
1770     const char *class = NULL;
1771     const char *provider = NULL;
1772     const char *type = NULL;
1773 
1774     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1775     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1776     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1777     free_xml(data);
1778 
1779     if (!output) {
1780         return NULL;
1781     }
1782 
1783     class = crm_element_value(output, F_LRMD_CLASS);
1784     provider = crm_element_value(output, F_LRMD_PROVIDER);
1785     type = crm_element_value(output, F_LRMD_TYPE);
1786 
1787     if (!class || !type) {
1788         free_xml(output);
1789         return NULL;
1790     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1791                && !provider) {
1792         free_xml(output);
1793         return NULL;
1794     }
1795 
1796     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1797     free_xml(output);
1798     return rsc_info;
1799 }
1800 
1801 void
1802 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1803 {
1804     if (op_info) {
1805         free(op_info->rsc_id);
1806         free(op_info->action);
1807         free(op_info->interval_ms_s);
1808         free(op_info->timeout_ms_s);
1809         free(op_info);
1810     }
1811 }
1812 
1813 static int
1814 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1815                            enum lrmd_call_options options, GList **output)
1816 {
1817     xmlNode *data = NULL;
1818     xmlNode *output_xml = NULL;
1819     int rc = pcmk_ok;
1820 
1821     if (output == NULL) {
1822         return -EINVAL;
1823     }
1824     *output = NULL;
1825 
1826     // Send request
1827     if (rsc_id) {
1828         data = create_xml_node(NULL, F_LRMD_RSC);
1829         crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1830         crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1831     }
1832     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1833                            timeout_ms, options, TRUE);
1834     if (data) {
1835         free_xml(data);
1836     }
1837 
1838     // Process reply
1839     if ((rc != pcmk_ok) || (output_xml == NULL)) {
1840         return rc;
1841     }
1842     for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
1843          (rsc_xml != NULL) && (rc == pcmk_ok);
1844          rsc_xml = crm_next_same_xml(rsc_xml)) {
1845 
1846         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1847         if (rsc_id == NULL) {
1848             crm_err("Could not parse recurring operation information from executor");
1849             continue;
1850         }
1851         for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
1852              op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
1853 
1854             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
1855 
1856             if (op_info == NULL) {
1857                 rc = -ENOMEM;
1858                 break;
1859             }
1860             op_info->rsc_id = strdup(rsc_id);
1861             op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
1862             op_info->interval_ms_s = crm_element_value_copy(op_xml,
1863                                                             F_LRMD_RSC_INTERVAL);
1864             op_info->timeout_ms_s = crm_element_value_copy(op_xml,
1865                                                            F_LRMD_TIMEOUT);
1866             *output = g_list_prepend(*output, op_info);
1867         }
1868     }
1869     free_xml(output_xml);
1870     return rc;
1871 }
1872 
1873 
1874 static void
1875 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
1876 {
1877     lrmd_private_t *native = lrmd->lrmd_private;
1878 
1879     native->callback = callback;
1880 }
1881 
1882 void
1883 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
1884 {
1885     lrmd_private_t *native = lrmd->lrmd_private;
1886 
1887     native->proxy_callback = callback;
1888     native->proxy_callback_userdata = userdata;
1889 }
1890 
1891 void
1892 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
1893 {
1894     lrmd_private_t *native = lrmd->lrmd_private;
1895 
1896     if (native->proxy_callback) {
1897         crm_log_xml_trace(msg, "PROXY_INBOUND");
1898         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
1899     }
1900 }
1901 
1902 int
1903 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
1904 {
1905     if (lrmd == NULL) {
1906         return -ENOTCONN;
1907     }
1908     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
1909 
1910     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
1911     return lrmd_send_xml_no_reply(lrmd, msg);
1912 }
1913 
1914 static int
1915 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
1916 {
1917     int rc = pcmk_ok;
1918     stonith_t *stonith_api = stonith_api_new();
1919 
1920     if (stonith_api == NULL) {
1921         crm_err("Could not get fence agent meta-data: API memory allocation failed");
1922         return -ENOMEM;
1923     }
1924 
1925     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
1926                                      provider, output, 0);
1927     if ((rc == pcmk_ok) && (*output == NULL)) {
1928         rc = -EIO;
1929     }
1930     stonith_api->cmds->free(stonith_api);
1931     return rc;
1932 }
1933 
1934 static int
1935 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
1936                       const char *type, char **output,
1937                       enum lrmd_call_options options)
1938 {
1939     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
1940                                            output, options, NULL);
1941 }
1942 
1943 static int
1944 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
1945                              const char *provider, const char *type,
1946                              char **output, enum lrmd_call_options options,
1947                              lrmd_key_value_t *params)
1948 {
1949     svc_action_t *action = NULL;
1950     GHashTable *params_table = NULL;
1951 
1952     if (!standard || !type) {
1953         lrmd_key_value_freeall(params);
1954         return -EINVAL;
1955     }
1956 
1957     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
1958         lrmd_key_value_freeall(params);
1959         return stonith_get_metadata(provider, type, output);
1960     }
1961 
1962     params_table = pcmk__strkey_table(free, free);
1963     for (const lrmd_key_value_t *param = params; param; param = param->next) {
1964         g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
1965     }
1966     action = services__create_resource_action(type, standard, provider, type,
1967                                               CRMD_ACTION_METADATA, 0,
1968                                               CRMD_METADATA_CALL_TIMEOUT,
1969                                               params_table, 0);
1970     lrmd_key_value_freeall(params);
1971 
1972     if (action == NULL) {
1973         return -ENOMEM;
1974     }
1975     if (action->rc != PCMK_OCF_UNKNOWN) {
1976         services_action_free(action);
1977         return -EINVAL;
1978     }
1979 
1980     if (!services_action_sync(action)) {
1981         crm_err("Failed to retrieve meta-data for %s:%s:%s",
1982                 standard, provider, type);
1983         services_action_free(action);
1984         return -EIO;
1985     }
1986 
1987     if (!action->stdout_data) {
1988         crm_err("Failed to receive meta-data for %s:%s:%s",
1989                 standard, provider, type);
1990         services_action_free(action);
1991         return -EIO;
1992     }
1993 
1994     *output = strdup(action->stdout_data);
1995     services_action_free(action);
1996 
1997     return pcmk_ok;
1998 }
1999 
2000 static int
2001 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2002               const char *userdata, guint interval_ms,
2003               int timeout,      /* ms */
2004               int start_delay,  /* ms */
2005               enum lrmd_call_options options, lrmd_key_value_t * params)
2006 {
2007     int rc = pcmk_ok;
2008     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2009     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2010     lrmd_key_value_t *tmp = NULL;
2011 
2012     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2013     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2014     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2015     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
2016     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2017     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2018     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
2019 
2020     for (tmp = params; tmp; tmp = tmp->next) {
2021         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2022     }
2023 
2024     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
2025     free_xml(data);
2026 
2027     lrmd_key_value_freeall(params);
2028     return rc;
2029 }
2030 
2031 /* timeout is in ms */
2032 static int
2033 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
2034                     int timeout, lrmd_key_value_t *params)
2035 {
2036     int rc = pcmk_ok;
2037     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
2038     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2039     lrmd_key_value_t *tmp = NULL;
2040 
2041     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2042     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
2043     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
2044     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2045 
2046     for (tmp = params; tmp; tmp = tmp->next) {
2047         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2048     }
2049 
2050     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2051                            lrmd_opt_notify_orig_only, TRUE);
2052     free_xml(data);
2053 
2054     lrmd_key_value_freeall(params);
2055     return rc;
2056 }
2057 
2058 static int
2059 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2060                 guint interval_ms)
2061 {
2062     int rc = pcmk_ok;
2063     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2064 
2065     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2066     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2067     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2068     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2069     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
2070     free_xml(data);
2071     return rc;
2072 }
2073 
2074 static int
2075 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
2076 {
2077     int rc = 0;
2078     stonith_t *stonith_api = stonith_api_new();
2079     stonith_key_value_t *stonith_resources = NULL;
2080     stonith_key_value_t *dIter = NULL;
2081 
2082     if (stonith_api == NULL) {
2083         crm_err("Could not list fence agents: API memory allocation failed");
2084         return -ENOMEM;
2085     }
2086     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
2087                                    &stonith_resources, 0);
2088     stonith_api->cmds->free(stonith_api);
2089 
2090     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
2091         rc++;
2092         if (resources) {
2093             *resources = lrmd_list_add(*resources, dIter->value);
2094         }
2095     }
2096 
2097     stonith_key_value_freeall(stonith_resources, 1, 0);
2098     return rc;
2099 }
2100 
2101 static int
2102 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
2103                      const char *provider)
2104 {
2105     int rc = 0;
2106     int stonith_count = 0; // Initially, whether to include stonith devices
2107 
2108     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2109         stonith_count = 1;
2110 
2111     } else {
2112         GList *gIter = NULL;
2113         GList *agents = resources_list_agents(class, provider);
2114 
2115         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
2116             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
2117             rc++;
2118         }
2119         g_list_free_full(agents, free);
2120 
2121         if (!class) {
2122             stonith_count = 1;
2123         }
2124     }
2125 
2126     if (stonith_count) {
2127         // Now, if stonith devices are included, how many there are
2128         stonith_count = list_stonith_agents(resources);
2129         if (stonith_count > 0) {
2130             rc += stonith_count;
2131         }
2132     }
2133     if (rc == 0) {
2134         crm_notice("No agents found for class %s", class);
2135         rc = -EPROTONOSUPPORT;
2136     }
2137     return rc;
2138 }
2139 
2140 static bool
2141 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
2142 {
2143     bool found = false;
2144     GList *agents = NULL;
2145     GList *gIter2 = NULL;
2146 
2147     agents = resources_list_agents(class, provider);
2148     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2149         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2150             found = true;
2151         }
2152     }
2153     g_list_free_full(agents, free);
2154     return found;
2155 }
2156 
2157 static int
2158 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
2159 {
2160     int rc = pcmk_ok;
2161     char *provider = NULL;
2162     GList *ocf_providers = NULL;
2163     GList *gIter = NULL;
2164 
2165     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2166 
2167     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2168         provider = gIter->data;
2169         if (!agent || does_provider_have_agent(agent, provider,
2170                                                PCMK_RESOURCE_CLASS_OCF)) {
2171             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2172             rc++;
2173         }
2174     }
2175 
2176     g_list_free_full(ocf_providers, free);
2177     return rc;
2178 }
2179 
2180 static int
2181 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
2182 {
2183     int rc = 0;
2184     GList *standards = NULL;
2185     GList *gIter = NULL;
2186 
2187     standards = resources_list_standards();
2188 
2189     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2190         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2191         rc++;
2192     }
2193 
2194     if (list_stonith_agents(NULL) > 0) {
2195         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2196         rc++;
2197     }
2198 
2199     g_list_free_full(standards, free);
2200     return rc;
2201 }
2202 
2203 /*!
2204  * \internal
2205  * \brief Create an executor API object
2206  *
2207  * \param[out] api       Will be set to newly created API object (it is the
2208  *                       caller's responsibility to free this value with
2209  *                       lrmd_api_delete() if this function succeeds)
2210  * \param[in]  nodename  If the object will be used for a remote connection,
2211  *                       the node name to use in cluster for remote executor
2212  * \param[in]  server    If the object will be used for a remote connection,
2213  *                       the resolvable host name to connect to
2214  * \param[in]  port      If the object will be used for a remote connection,
2215  *                       port number on \p server to connect to
2216  *
2217  * \return Standard Pacemaker return code
2218  * \note If the caller leaves one of \p nodename or \p server NULL, the other's
2219  *       value will be used for both. If the caller leaves both NULL, an API
2220  *       object will be created for a local executor connection.
2221  */
2222 int
2223 lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
     /*  */
2224 {
2225     lrmd_private_t *pvt = NULL;
2226 
2227     if (api == NULL) {
2228         return EINVAL;
2229     }
2230     *api = NULL;
2231 
2232     // Allocate all memory needed
2233 
2234     *api = calloc(1, sizeof(lrmd_t));
2235     if (*api == NULL) {
2236         return ENOMEM;
2237     }
2238 
2239     pvt = calloc(1, sizeof(lrmd_private_t));
2240     if (pvt == NULL) {
2241         lrmd_api_delete(*api);
2242         *api = NULL;
2243         return ENOMEM;
2244     }
2245     (*api)->lrmd_private = pvt;
2246 
2247     // @TODO Do we need to do this for local connections?
2248     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2249 
2250     (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2251 
2252     if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
2253         lrmd_api_delete(*api);
2254         *api = NULL;
2255         return ENOMEM;
2256     }
2257 
2258     // Set methods
2259     (*api)->cmds->connect = lrmd_api_connect;
2260     (*api)->cmds->connect_async = lrmd_api_connect_async;
2261     (*api)->cmds->is_connected = lrmd_api_is_connected;
2262     (*api)->cmds->poke_connection = lrmd_api_poke_connection;
2263     (*api)->cmds->disconnect = lrmd_api_disconnect;
2264     (*api)->cmds->register_rsc = lrmd_api_register_rsc;
2265     (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2266     (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2267     (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2268     (*api)->cmds->set_callback = lrmd_api_set_callback;
2269     (*api)->cmds->get_metadata = lrmd_api_get_metadata;
2270     (*api)->cmds->exec = lrmd_api_exec;
2271     (*api)->cmds->cancel = lrmd_api_cancel;
2272     (*api)->cmds->list_agents = lrmd_api_list_agents;
2273     (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2274     (*api)->cmds->list_standards = lrmd_api_list_standards;
2275     (*api)->cmds->exec_alert = lrmd_api_exec_alert;
2276     (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2277 
2278     if ((nodename == NULL) && (server == NULL)) {
2279         pvt->type = pcmk__client_ipc;
2280     } else {
2281 #ifdef HAVE_GNUTLS_GNUTLS_H
2282         if (nodename == NULL) {
2283             nodename = server;
2284         } else if (server == NULL) {
2285             server = nodename;
2286         }
2287         pvt->type = pcmk__client_tls;
2288         pvt->remote_nodename = strdup(nodename);
2289         pvt->server = strdup(server);
2290         if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
2291             lrmd_api_delete(*api);
2292             *api = NULL;
2293             return ENOMEM;
2294         }
2295         pvt->port = port;
2296         if (pvt->port == 0) {
2297             pvt->port = crm_default_remote_port();
2298         }
2299 #else
2300         crm_err("Cannot communicate with Pacemaker Remote "
2301                 "because GnuTLS is not enabled for this build");
2302         lrmd_api_delete(*api);
2303         *api = NULL;
2304         return EOPNOTSUPP;
2305 #endif
2306     }
2307     return pcmk_rc_ok;
2308 }
2309 
2310 lrmd_t *
2311 lrmd_api_new(void)
     /*  */
2312 {
2313     lrmd_t *api = NULL;
2314 
2315     CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
2316     return api;
2317 }
2318 
2319 lrmd_t *
2320 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2321 {
2322     lrmd_t *api = NULL;
2323 
2324     CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
2325     return api;
2326 }
2327 
2328 void
2329 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2330 {
2331     if (lrmd == NULL) {
2332         return;
2333     }
2334     if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
2335         if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
2336             lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
2337         }
2338         free(lrmd->cmds);
2339     }
2340     if (lrmd->lrmd_private != NULL) {
2341         lrmd_private_t *native = lrmd->lrmd_private;
2342 
2343 #ifdef HAVE_GNUTLS_GNUTLS_H
2344         free(native->server);
2345 #endif
2346         free(native->remote_nodename);
2347         free(native->remote);
2348         free(native->token);
2349         free(native->peer_version);
2350         free(lrmd->lrmd_private);
2351     }
2352     free(lrmd);
2353 }
2354 
2355 struct metadata_cb {
2356      void (*callback)(int pid, const pcmk__action_result_t *result,
2357                       void *user_data);
2358      void *user_data;
2359 };
2360 
2361 /*!
2362  * \internal
2363  * \brief Process asynchronous metadata completion
2364  *
2365  * \param[in,out] action  Metadata action that completed
2366  */
2367 static void
2368 metadata_complete(svc_action_t *action)
     /*  */
2369 {
2370     struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
2371     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2372 
2373     pcmk__set_result(&result, action->rc, action->status,
2374                      services__exit_reason(action));
2375     pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
2376 
2377     metadata_cb->callback(0, &result, metadata_cb->user_data);
2378     result.action_stdout = NULL; // Prevent free, because action owns it
2379     result.action_stderr = NULL; // Prevent free, because action owns it
2380     pcmk__reset_result(&result);
2381     free(metadata_cb);
2382 }
2383 
2384 /*!
2385  * \internal
2386  * \brief Retrieve agent metadata asynchronously
2387  *
2388  * \param[in]     rsc        Resource agent specification
2389  * \param[in]     callback   Function to call with result (this will always be
2390  *                           called, whether by this function directly or later
2391  *                           via the main loop, and on success the metadata will
2392  *                           be in its result argument's action_stdout)
2393  * \param[in,out] user_data  User data to pass to callback
2394  *
2395  * \return Standard Pacemaker return code
2396  * \note This function is not a lrmd_api_operations_t method because it does not
2397  *       need an lrmd_t object and does not go through the executor, but
2398  *       executes the agent directly.
2399  */
2400 int
2401 lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
     /*  */
2402                      void (*callback)(int pid,
2403                                       const pcmk__action_result_t *result,
2404                                       void *user_data),
2405                      void *user_data)
2406 {
2407     svc_action_t *action = NULL;
2408     struct metadata_cb *metadata_cb = NULL;
2409     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2410 
2411     CRM_CHECK(callback != NULL, return EINVAL);
2412 
2413     if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
2414         pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
2415                          PCMK_EXEC_ERROR_FATAL,
2416                          "Invalid resource specification");
2417         callback(0, &result, user_data);
2418         pcmk__reset_result(&result);
2419         return EINVAL;
2420     }
2421 
2422     if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
2423         return stonith__metadata_async(rsc->type,
2424                                        CRMD_METADATA_CALL_TIMEOUT / 1000,
2425                                        callback, user_data);
2426     }
2427 
2428     action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
2429                                               rsc->standard, rsc->provider,
2430                                               rsc->type, CRMD_ACTION_METADATA,
2431                                               0, CRMD_METADATA_CALL_TIMEOUT,
2432                                               NULL, 0);
2433     if (action == NULL) {
2434         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2435                          "Out of memory");
2436         callback(0, &result, user_data);
2437         pcmk__reset_result(&result);
2438         return ENOMEM;
2439     }
2440     if (action->rc != PCMK_OCF_UNKNOWN) {
2441         pcmk__set_result(&result, action->rc, action->status,
2442                          services__exit_reason(action));
2443         callback(0, &result, user_data);
2444         pcmk__reset_result(&result);
2445         services_action_free(action);
2446         return EINVAL;
2447     }
2448 
2449     action->cb_data = calloc(1, sizeof(struct metadata_cb));
2450     if (action->cb_data == NULL) {
2451         services_action_free(action);
2452         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2453                          "Out of memory");
2454         callback(0, &result, user_data);
2455         pcmk__reset_result(&result);
2456         return ENOMEM;
2457     }
2458 
2459     metadata_cb = (struct metadata_cb *) action->cb_data;
2460     metadata_cb->callback = callback;
2461     metadata_cb->user_data = user_data;
2462     if (!services_action_async(action, metadata_complete)) {
2463         services_action_free(action);
2464         return pcmk_rc_error; // @TODO Derive from action->rc and ->status
2465     }
2466 
2467     // The services library has taken responsibility for action
2468     return pcmk_rc_ok;
2469 }
2470 
2471 /*!
2472  * \internal
2473  * \brief Set the result of an executor event
2474  *
2475  * \param[in,out] event        Executor event to set
2476  * \param[in]     rc           OCF exit status of event
2477  * \param[in]     op_status    Executor status of event
2478  * \param[in]     exit_reason  Human-friendly description of event
2479  */
2480 void
2481 lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
     /*  */
2482                  const char *exit_reason)
2483 {
2484     if (event == NULL) {
2485         return;
2486     }
2487 
2488     event->rc = rc;
2489     event->op_status = op_status;
2490     pcmk__str_update((char **) &event->exit_reason, exit_reason);
2491 }
2492 
2493 /*!
2494  * \internal
2495  * \brief Clear an executor event's exit reason, output, and error output
2496  *
2497  * \param[in,out] event  Executor event to reset
2498  */
2499 void
2500 lrmd__reset_result(lrmd_event_data_t *event)
     /*  */
2501 {
2502     if (event == NULL) {
2503         return;
2504     }
2505 
2506     free((void *) event->exit_reason);
2507     event->exit_reason = NULL;
2508 
2509     free((void *) event->output);
2510     event->output = NULL;
2511 }
2512 
2513 /*!
2514  * \internal
2515  * \brief Get the uptime of a remote resource connection
2516  *
2517  * When the cluster connects to a remote resource, part of that resource's
2518  * handshake includes the uptime of the remote resource's connection.  This
2519  * uptime is stored in the lrmd_t object.
2520  *
2521  * \return The connection's uptime, or -1 if unknown
2522  */
2523 time_t
2524 lrmd__uptime(lrmd_t *lrmd)
     /*  */
2525 {
2526     lrmd_private_t *native = lrmd->lrmd_private;
2527 
2528     if (native->remote == NULL) {
2529         return -1;
2530     } else {
2531         return native->remote->uptime;
2532     }
2533 }
/* */
root/lib/lrmd/lrmd_client.c

DEFINITIONS
