lib/lrmd/lrmd_client.c

/* */
This source file includes following definitions.
lrmd_list_add
lrmd_list_freeall
lrmd_key_value_add
lrmd_key_value_freeall
lrmd_new_event
lrmd_copy_event
lrmd_free_event
lrmd_dispatch_internal
lrmd_ipc_dispatch
lrmd_free_xml
lrmd_tls_connected
lrmd_tls_dispatch
lrmd_poll
lrmd_dispatch
lrmd_create_op
lrmd_ipc_connection_destroy
lrmd_tls_connection_destroy
lrmd_tls_send_msg
lrmd_tls_recv_reply
lrmd_tls_send
lrmd_tls_send_recv
lrmd_send_xml
lrmd_send_xml_no_reply
lrmd_api_is_connected
lrmd_send_command
lrmd_api_poke_connection
remote_proxy_check
lrmd_handshake
lrmd_ipc_connect
copy_gnutls_datum
clear_gnutls_datum
set_key
lrmd_tls_set_key
lrmd_gnutls_global_init
report_async_connection_result
lrmd__tls_client_handshake
lrmd_tcp_connect_cb
lrmd_tls_connect_async
lrmd_tls_connect
lrmd_api_connect
lrmd_api_connect_async
lrmd_ipc_disconnect
lrmd_tls_disconnect
lrmd_api_disconnect
lrmd_api_register_rsc
lrmd_api_unregister_rsc
lrmd_new_rsc_info
lrmd_copy_rsc_info
lrmd_free_rsc_info
lrmd_api_get_rsc_info
lrmd_free_op_info
lrmd_api_get_recurring_ops
lrmd_api_set_callback
lrmd_internal_set_proxy_callback
lrmd_internal_proxy_dispatch
lrmd_internal_proxy_send
stonith_get_metadata
lrmd_api_get_metadata
lrmd_api_get_metadata_params
lrmd_api_exec
lrmd_api_exec_alert
lrmd_api_cancel
list_stonith_agents
lrmd_api_list_agents
does_provider_have_agent
lrmd_api_list_ocf_providers
lrmd_api_list_standards
lrmd_api_new
lrmd_remote_api_new
lrmd_api_delete
   1 /*
   2  * Copyright 2012-2020 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 
  20 #include <sys/types.h>
  21 #include <sys/wait.h>
  22 
  23 #include <glib.h>
  24 #include <dirent.h>
  25 
  26 #include <crm/crm.h>
  27 #include <crm/lrmd.h>
  28 #include <crm/lrmd_internal.h>
  29 #include <crm/services.h>
  30 #include <crm/common/mainloop.h>
  31 #include <crm/common/ipc_internal.h>
  32 #include <crm/common/remote_internal.h>
  33 #include <crm/msg_xml.h>
  34 
  35 #include <crm/stonith-ng.h>
  36 
  37 #ifdef HAVE_GNUTLS_GNUTLS_H
  38 #  undef KEYFILE
  39 #  include <gnutls/gnutls.h>
  40 #endif
  41 
  42 #include <sys/socket.h>
  43 #include <netinet/in.h>
  44 #include <netinet/ip.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47 
  48 #define MAX_TLS_RECV_WAIT 10000
  49 
  50 CRM_TRACE_INIT_DATA(lrmd);
  51 
  52 static int lrmd_api_disconnect(lrmd_t * lrmd);
  53 static int lrmd_api_is_connected(lrmd_t * lrmd);
  54 
  55 /* IPC proxy functions */
  56 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  57 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  58 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  59 
  60 #ifdef HAVE_GNUTLS_GNUTLS_H
  61 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
  62 gnutls_psk_client_credentials_t psk_cred_s;
  63 int lrmd_tls_set_key(gnutls_datum_t * key);
  64 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  65 static int global_remote_msg_id = 0;
  66 static void lrmd_tls_connection_destroy(gpointer userdata);
  67 #endif
  68 
  69 typedef struct lrmd_private_s {
  70     uint64_t type;
  71     char *token;
  72     mainloop_io_t *source;
  73 
  74     /* IPC parameters */
  75     crm_ipc_t *ipc;
  76 
  77     pcmk__remote_t *remote;
  78 
  79     /* Extra TLS parameters */
  80     char *remote_nodename;
  81 #ifdef HAVE_GNUTLS_GNUTLS_H
  82     char *server;
  83     int port;
  84     gnutls_psk_client_credentials_t psk_cred_c;
  85 
  86     /* while the async connection is occurring, this is the id
  87      * of the connection timeout timer. */
  88     int async_timer;
  89     int sock;
  90     /* since tls requires a round trip across the network for a
  91      * request/reply, there are times where we just want to be able
  92      * to send a request from the client and not wait around (or even care
  93      * about) what the reply is. */
  94     int expected_late_replies;
  95     GList *pending_notify;
  96     crm_trigger_t *process_notify;
  97 #endif
  98 
  99     lrmd_event_callback callback;
 100 
 101     /* Internal IPC proxy msg passing for remote guests */
 102     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 103     void *proxy_callback_userdata;
 104     char *peer_version;
 105 } lrmd_private_t;
 106 
 107 static lrmd_list_t *
 108 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 109 {
 110     lrmd_list_t *p, *end;
 111 
 112     p = calloc(1, sizeof(lrmd_list_t));
 113     p->val = strdup(value);
 114 
 115     end = head;
 116     while (end && end->next) {
 117         end = end->next;
 118     }
 119 
 120     if (end) {
 121         end->next = p;
 122     } else {
 123         head = p;
 124     }
 125 
 126     return head;
 127 }
 128 
 129 void
 130 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 131 {
 132     lrmd_list_t *p;
 133 
 134     while (head) {
 135         char *val = (char *)head->val;
 136 
 137         p = head->next;
 138         free(val);
 139         free(head);
 140         head = p;
 141     }
 142 }
 143 
 144 lrmd_key_value_t *
 145 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 146 {
 147     lrmd_key_value_t *p, *end;
 148 
 149     p = calloc(1, sizeof(lrmd_key_value_t));
 150     p->key = strdup(key);
 151     p->value = strdup(value);
 152 
 153     end = head;
 154     while (end && end->next) {
 155         end = end->next;
 156     }
 157 
 158     if (end) {
 159         end->next = p;
 160     } else {
 161         head = p;
 162     }
 163 
 164     return head;
 165 }
 166 
 167 void
 168 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 169 {
 170     lrmd_key_value_t *p;
 171 
 172     while (head) {
 173         p = head->next;
 174         free(head->key);
 175         free(head->value);
 176         free(head);
 177         head = p;
 178     }
 179 }
 180 
 181 /*!
 182  * Create a new lrmd_event_data_t object
 183  *
 184  * \param[in] rsc_id       ID of resource involved in event
 185  * \param[in] task         Action name
 186  * \param[in] interval_ms  Action interval
 187  *
 188  * \return Newly allocated and initialized lrmd_event_data_t
 189  * \note This functions asserts on memory errors, so the return value is
 190  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 191  *       result with lrmd_free_event().
 192  */
 193 lrmd_event_data_t *
 194 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 195 {
 196     lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
 197 
 198     CRM_ASSERT(event != NULL);
 199     if (rsc_id != NULL) {
 200         event->rsc_id = strdup(rsc_id);
 201         CRM_ASSERT(event->rsc_id != NULL);
 202     }
 203     if (task != NULL) {
 204         event->op_type = strdup(task);
 205         CRM_ASSERT(event->op_type != NULL);
 206     }
 207     event->interval_ms = interval_ms;
 208     return event;
 209 }
 210 
 211 lrmd_event_data_t *
 212 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 213 {
 214     lrmd_event_data_t *copy = NULL;
 215 
 216     copy = calloc(1, sizeof(lrmd_event_data_t));
 217 
 218     /* This will get all the int values.
 219      * we just have to be careful not to leave any
 220      * dangling pointers to strings. */
 221     memcpy(copy, event, sizeof(lrmd_event_data_t));
 222 
 223     copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
 224     copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
 225     copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
 226     copy->output = event->output ? strdup(event->output) : NULL;
 227     copy->exit_reason = event->exit_reason ? strdup(event->exit_reason) : NULL;
 228     copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;
 229     copy->params = crm_str_table_dup(event->params);
 230 
 231     return copy;
 232 }
 233 
 234 void
 235 lrmd_free_event(lrmd_event_data_t * event)
     /*  */
 236 {
 237     if (!event) {
 238         return;
 239     }
 240 
 241     /* free gives me grief if i try to cast */
 242     free((char *)event->rsc_id);
 243     free((char *)event->op_type);
 244     free((char *)event->user_data);
 245     free((char *)event->output);
 246     free((char *)event->exit_reason);
 247     free((char *)event->remote_nodename);
 248     if (event->params) {
 249         g_hash_table_destroy(event->params);
 250     }
 251     free(event);
 252 }
 253 
 254 static int
 255 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 256 {
 257     const char *type;
 258     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
 259     lrmd_private_t *native = lrmd->lrmd_private;
 260     lrmd_event_data_t event = { 0, };
 261 
 262     if (proxy_session != NULL) {
 263         /* this is proxy business */
 264         lrmd_internal_proxy_dispatch(lrmd, msg);
 265         return 1;
 266     } else if (!native->callback) {
 267         /* no callback set */
 268         crm_trace("notify event received but client has not set callback");
 269         return 1;
 270     }
 271 
 272     event.remote_nodename = native->remote_nodename;
 273     type = crm_element_value(msg, F_LRMD_OPERATION);
 274     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
 275     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 276 
 277     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 278         event.type = lrmd_event_register;
 279     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 280         event.type = lrmd_event_unregister;
 281     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 282         time_t epoch = 0;
 283 
 284         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
 285         crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
 286         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
 287         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
 288         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
 289         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 290 
 291         crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
 292         event.t_run = (unsigned int) epoch;
 293 
 294         crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
 295         event.t_rcchange = (unsigned int) epoch;
 296 
 297         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
 298         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 299 
 300         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
 301         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
 302         event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
 303         event.exit_reason = crm_element_value(msg, F_LRMD_RSC_EXIT_REASON);
 304         event.type = lrmd_event_exec_complete;
 305 
 306         event.params = xml2list(msg);
 307     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 308         event.type = lrmd_event_new_client;
 309     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 310         event.type = lrmd_event_poke;
 311     } else {
 312         return 1;
 313     }
 314 
 315     crm_trace("op %s notify event received", type);
 316     native->callback(&event);
 317 
 318     if (event.params) {
 319         g_hash_table_destroy(event.params);
 320     }
 321     return 1;
 322 }
 323 
 324 static int
 325 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 326 {
 327     lrmd_t *lrmd = userdata;
 328     lrmd_private_t *native = lrmd->lrmd_private;
 329     xmlNode *msg;
 330     int rc;
 331 
 332     if (!native->callback) {
 333         /* no callback set */
 334         return 1;
 335     }
 336 
 337     msg = string2xml(buffer);
 338     rc = lrmd_dispatch_internal(lrmd, msg);
 339     free_xml(msg);
 340     return rc;
 341 }
 342 
 343 #ifdef HAVE_GNUTLS_GNUTLS_H
 344 static void
 345 lrmd_free_xml(gpointer userdata)
     /*  */
 346 {
 347     free_xml((xmlNode *) userdata);
 348 }
 349 
 350 static int
 351 lrmd_tls_connected(lrmd_t * lrmd)
     /*  */
 352 {
 353     lrmd_private_t *native = lrmd->lrmd_private;
 354 
 355     if (native->remote->tls_session) {
 356         return TRUE;
 357     }
 358 
 359     return FALSE;
 360 }
 361 
 362 static int
 363 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 364 {
 365     lrmd_t *lrmd = userdata;
 366     lrmd_private_t *native = lrmd->lrmd_private;
 367     xmlNode *xml = NULL;
 368     int rc = pcmk_rc_ok;
 369 
 370     if (lrmd_tls_connected(lrmd) == FALSE) {
 371         crm_trace("TLS dispatch triggered after disconnect");
 372         return 0;
 373     }
 374 
 375     crm_trace("TLS dispatch triggered");
 376 
 377     /* First check if there are any pending notifies to process that came
 378      * while we were waiting for replies earlier. */
 379     if (native->pending_notify) {
 380         GList *iter = NULL;
 381 
 382         crm_trace("Processing pending notifies");
 383         for (iter = native->pending_notify; iter; iter = iter->next) {
 384             lrmd_dispatch_internal(lrmd, iter->data);
 385         }
 386         g_list_free_full(native->pending_notify, lrmd_free_xml);
 387         native->pending_notify = NULL;
 388     }
 389 
 390     /* Next read the current buffer and see if there are any messages to handle. */
 391     switch (pcmk__remote_ready(native->remote, 0)) {
 392         case pcmk_rc_ok:
 393             rc = pcmk__read_remote_message(native->remote, -1);
 394             xml = pcmk__remote_message_xml(native->remote);
 395             break;
 396         case ETIME:
 397             // Nothing to read, check if a full message is already in buffer
 398             xml = pcmk__remote_message_xml(native->remote);
 399             break;
 400         default:
 401             rc = ENOTCONN;
 402             break;
 403     }
 404     while (xml) {
 405         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 406         if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 407             lrmd_dispatch_internal(lrmd, xml);
 408         } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 409             if (native->expected_late_replies > 0) {
 410                 native->expected_late_replies--;
 411             } else {
 412                 int reply_id = 0;
 413                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
 414                 /* if this happens, we want to know about it */
 415                 crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 416             }
 417         }
 418         free_xml(xml);
 419         xml = pcmk__remote_message_xml(native->remote);
 420     }
 421 
 422     if (rc == ENOTCONN) {
 423         crm_info("Lost %s executor connection while reading data",
 424                  (native->remote_nodename? native->remote_nodename : "local"));
 425         lrmd_tls_disconnect(lrmd);
 426         return 0;
 427     }
 428     return 1;
 429 }
 430 #endif
 431 
 432 /* Not used with mainloop */
 433 int
 434 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 435 {
 436     lrmd_private_t *native = lrmd->lrmd_private;
 437 
 438     switch (native->type) {
 439         case pcmk__client_ipc:
 440             return crm_ipc_ready(native->ipc);
 441 
 442 #ifdef HAVE_GNUTLS_GNUTLS_H
 443         case pcmk__client_tls:
 444             if (native->pending_notify) {
 445                 return 1;
 446             } else {
 447                 int rc = pcmk__remote_ready(native->remote, 0);
 448 
 449                 switch (rc) {
 450                     case pcmk_rc_ok:
 451                         return 1;
 452                     case ETIME:
 453                         return 0;
 454                     default:
 455                         return pcmk_rc2legacy(rc);
 456                 }
 457             }
 458 #endif
 459         default:
 460             crm_err("Unsupported connection type: %d", native->type);
 461     }
 462 
 463     return 0;
 464 }
 465 
 466 /* Not used with mainloop */
 467 bool
 468 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 469 {
 470     lrmd_private_t *private = NULL;
 471 
 472     CRM_ASSERT(lrmd != NULL);
 473 
 474     private = lrmd->lrmd_private;
 475     switch (private->type) {
 476         case pcmk__client_ipc:
 477             while (crm_ipc_ready(private->ipc)) {
 478                 if (crm_ipc_read(private->ipc) > 0) {
 479                     const char *msg = crm_ipc_buffer(private->ipc);
 480 
 481                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 482                 }
 483             }
 484             break;
 485 #ifdef HAVE_GNUTLS_GNUTLS_H
 486         case pcmk__client_tls:
 487             lrmd_tls_dispatch(lrmd);
 488             break;
 489 #endif
 490         default:
 491             crm_err("Unsupported connection type: %d", private->type);
 492     }
 493 
 494     if (lrmd_api_is_connected(lrmd) == FALSE) {
 495         crm_err("Connection closed");
 496         return FALSE;
 497     }
 498 
 499     return TRUE;
 500 }
 501 
 502 static xmlNode *
 503 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 504                enum lrmd_call_options options)
 505 {
 506     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 507 
 508     CRM_CHECK(op_msg != NULL, return NULL);
 509     CRM_CHECK(token != NULL, return NULL);
 510 
 511     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
 512     crm_xml_add(op_msg, F_TYPE, T_LRMD);
 513     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
 514     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
 515     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
 516     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 517 
 518     if (data != NULL) {
 519         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
 520     }
 521 
 522     crm_trace("Created executor %s command with call options %.8lx (%d)",
 523               op, (long)options, options);
 524     return op_msg;
 525 }
 526 
 527 static void
 528 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 529 {
 530     lrmd_t *lrmd = userdata;
 531     lrmd_private_t *native = lrmd->lrmd_private;
 532 
 533     crm_info("IPC connection destroyed");
 534 
 535     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 536     native->ipc = NULL;
 537     native->source = NULL;
 538 
 539     if (native->callback) {
 540         lrmd_event_data_t event = { 0, };
 541         event.type = lrmd_event_disconnect;
 542         event.remote_nodename = native->remote_nodename;
 543         native->callback(&event);
 544     }
 545 }
 546 
 547 #ifdef HAVE_GNUTLS_GNUTLS_H
 548 static void
 549 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 550 {
 551     lrmd_t *lrmd = userdata;
 552     lrmd_private_t *native = lrmd->lrmd_private;
 553 
 554     crm_info("TLS connection destroyed");
 555 
 556     if (native->remote->tls_session) {
 557         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 558         gnutls_deinit(*native->remote->tls_session);
 559         gnutls_free(native->remote->tls_session);
 560     }
 561     if (native->psk_cred_c) {
 562         gnutls_psk_free_client_credentials(native->psk_cred_c);
 563     }
 564     if (native->sock) {
 565         close(native->sock);
 566     }
 567     if (native->process_notify) {
 568         mainloop_destroy_trigger(native->process_notify);
 569         native->process_notify = NULL;
 570     }
 571     if (native->pending_notify) {
 572         g_list_free_full(native->pending_notify, lrmd_free_xml);
 573         native->pending_notify = NULL;
 574     }
 575 
 576     free(native->remote->buffer);
 577     native->remote->buffer = NULL;
 578     native->source = 0;
 579     native->sock = 0;
 580     native->psk_cred_c = NULL;
 581     native->remote->tls_session = NULL;
 582     native->sock = 0;
 583 
 584     if (native->callback) {
 585         lrmd_event_data_t event = { 0, };
 586         event.remote_nodename = native->remote_nodename;
 587         event.type = lrmd_event_disconnect;
 588         native->callback(&event);
 589     }
 590     return;
 591 }
 592 
 593 // \return Standard Pacemaker return code
 594 int
 595 lrmd_tls_send_msg(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 596                   const char *msg_type)
 597 {
 598     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
 599     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
 600     return pcmk__remote_send_xml(session, msg);
 601 }
 602 
 603 static xmlNode *
 604 lrmd_tls_recv_reply(lrmd_t * lrmd, int total_timeout, int expected_reply_id, int *disconnected)
     /*  */
 605 {
 606     lrmd_private_t *native = lrmd->lrmd_private;
 607     xmlNode *xml = NULL;
 608     time_t start = time(NULL);
 609     const char *msg_type = NULL;
 610     int reply_id = 0;
 611     int remaining_timeout = 0;
 612 
 613     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 614      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 615     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 616         total_timeout = MAX_TLS_RECV_WAIT;
 617     }
 618 
 619     while (!xml) {
 620 
 621         xml = pcmk__remote_message_xml(native->remote);
 622         if (!xml) {
 623             /* read some more off the tls buffer if we still have time left. */
 624             if (remaining_timeout) {
 625                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 626             } else {
 627                 remaining_timeout = total_timeout;
 628             }
 629             if (remaining_timeout <= 0) {
 630                 crm_err("Never received the expected reply during the timeout period, disconnecting.");
 631                 *disconnected = TRUE;
 632                 return NULL;
 633             }
 634 
 635             if (pcmk__read_remote_message(native->remote,
 636                                           remaining_timeout) == ENOTCONN) {
 637                 *disconnected = TRUE;
 638             } else {
 639                 *disconnected = FALSE;
 640             }
 641             xml = pcmk__remote_message_xml(native->remote);
 642             if (!xml) {
 643                 crm_err("Unable to receive expected reply, disconnecting.");
 644                 *disconnected = TRUE;
 645                 return NULL;
 646             } else if (*disconnected) {
 647                 return NULL;
 648             }
 649         }
 650 
 651         CRM_ASSERT(xml != NULL);
 652 
 653         crm_element_value_int(xml, F_LRMD_REMOTE_MSG_ID, &reply_id);
 654         msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 655 
 656         if (!msg_type) {
 657             crm_err("Empty msg type received while waiting for reply");
 658             free_xml(xml);
 659             xml = NULL;
 660         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 661             /* got a notify while waiting for reply, trigger the notify to be processed later */
 662             crm_info("queueing notify");
 663             native->pending_notify = g_list_append(native->pending_notify, xml);
 664             if (native->process_notify) {
 665                 crm_info("notify trigger set.");
 666                 mainloop_set_trigger(native->process_notify);
 667             }
 668             xml = NULL;
 669         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 670             /* msg isn't a reply, make some noise */
 671             crm_err("Expected a reply, got %s", msg_type);
 672             free_xml(xml);
 673             xml = NULL;
 674         } else if (reply_id != expected_reply_id) {
 675             if (native->expected_late_replies > 0) {
 676                 native->expected_late_replies--;
 677             } else {
 678                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 679             }
 680             free_xml(xml);
 681             xml = NULL;
 682         }
 683     }
 684 
 685     if (native->remote->buffer && native->process_notify) {
 686         mainloop_set_trigger(native->process_notify);
 687     }
 688 
 689     return xml;
 690 }
 691 
 692 static int
 693 lrmd_tls_send(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 694 {
 695     int rc = 0;
 696     lrmd_private_t *native = lrmd->lrmd_private;
 697 
 698     global_remote_msg_id++;
 699     if (global_remote_msg_id <= 0) {
 700         global_remote_msg_id = 1;
 701     }
 702 
 703     rc = lrmd_tls_send_msg(native->remote, msg, global_remote_msg_id, "request");
 704     if (rc != pcmk_rc_ok) {
 705         crm_err("Disconnecting because TLS message could not be sent to "
 706                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 707         lrmd_tls_disconnect(lrmd);
 708         return -ENOTCONN;
 709     }
 710     return pcmk_ok;
 711 }
 712 
 713 static int
 714 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 715 {
 716     int rc = 0;
 717     int disconnected = 0;
 718     xmlNode *xml = NULL;
 719 
 720     if (lrmd_tls_connected(lrmd) == FALSE) {
 721         return -1;
 722     }
 723 
 724     rc = lrmd_tls_send(lrmd, msg);
 725     if (rc < 0) {
 726         return rc;
 727     }
 728 
 729     xml = lrmd_tls_recv_reply(lrmd, timeout, global_remote_msg_id, &disconnected);
 730 
 731     if (disconnected) {
 732         crm_err("Pacemaker Remote disconnected while waiting for reply to request id %d",
 733                 global_remote_msg_id);
 734         lrmd_tls_disconnect(lrmd);
 735         rc = -ENOTCONN;
 736     } else if (!xml) {
 737         crm_err("Did not receive reply from Pacemaker Remote for request id %d (timeout %dms)",
 738                 global_remote_msg_id, timeout);
 739         rc = -ECOMM;
 740     }
 741 
 742     if (reply) {
 743         *reply = xml;
 744     } else {
 745         free_xml(xml);
 746     }
 747 
 748     return rc;
 749 }
 750 #endif
 751 
 752 static int
 753 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 754 {
 755     int rc = -1;
 756     lrmd_private_t *native = lrmd->lrmd_private;
 757 
 758     switch (native->type) {
 759         case pcmk__client_ipc:
 760             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 761             break;
 762 #ifdef HAVE_GNUTLS_GNUTLS_H
 763         case pcmk__client_tls:
 764             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 765             break;
 766 #endif
 767         default:
 768             crm_err("Unsupported connection type: %d", native->type);
 769     }
 770 
 771     return rc;
 772 }
 773 
 774 static int
 775 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 776 {
 777     int rc = -1;
 778     lrmd_private_t *native = lrmd->lrmd_private;
 779 
 780     switch (native->type) {
 781         case pcmk__client_ipc:
 782             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 783             break;
 784 #ifdef HAVE_GNUTLS_GNUTLS_H
 785         case pcmk__client_tls:
 786             rc = lrmd_tls_send(lrmd, msg);
 787             if (rc == pcmk_ok) {
 788                 /* we don't want to wait around for the reply, but
 789                  * since the request/reply protocol needs to behave the same
 790                  * as libqb, a reply will eventually come later anyway. */
 791                 native->expected_late_replies++;
 792             }
 793             break;
 794 #endif
 795         default:
 796             crm_err("Unsupported connection type: %d", native->type);
 797     }
 798 
 799     return rc;
 800 }
 801 
 802 static int
 803 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 804 {
 805     lrmd_private_t *native = lrmd->lrmd_private;
 806 
 807     switch (native->type) {
 808         case pcmk__client_ipc:
 809             return crm_ipc_connected(native->ipc);
 810 #ifdef HAVE_GNUTLS_GNUTLS_H
 811         case pcmk__client_tls:
 812             return lrmd_tls_connected(lrmd);
 813 #endif
 814         default:
 815             crm_err("Unsupported connection type: %d", native->type);
 816     }
 817 
 818     return 0;
 819 }
 820 
 821 /*!
 822  * \internal
 823  * \brief Send a prepared API command to the executor
 824  *
 825  * \param[in]  lrmd          Existing connection to the executor
 826  * \param[in]  op            Name of API command to send
 827  * \param[in]  data          Command data XML to add to the sent command
 828  * \param[out] output_data   If expecting a reply, it will be stored here
 829  * \param[in]  timeout       Timeout in milliseconds (if 0, defaults to
 830  *                           a sensible value per the type of connection,
 831  *                           standard vs. pacemaker remote);
 832  *                           also propagated to the command XML
 833  * \param[in]  call_options  Call options to pass to server when sending
 834  * \param[in]  expect_reply  If TRUE, wait for a reply from the server;
 835  *                           must be TRUE for IPC (as opposed to TLS) clients
 836  *
 837  * \return pcmk_ok on success, -errno on error
 838  */
 839 static int
 840 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 841                   xmlNode **output_data, int timeout,
 842                   enum lrmd_call_options options, gboolean expect_reply)
 843 {
 844     int rc = pcmk_ok;
 845     lrmd_private_t *native = lrmd->lrmd_private;
 846     xmlNode *op_msg = NULL;
 847     xmlNode *op_reply = NULL;
 848 
 849     if (!lrmd_api_is_connected(lrmd)) {
 850         return -ENOTCONN;
 851     }
 852 
 853     if (op == NULL) {
 854         crm_err("No operation specified");
 855         return -EINVAL;
 856     }
 857 
 858     CRM_CHECK(native->token != NULL,;
 859         );
 860     crm_trace("Sending %s op to executor", op);
 861 
 862     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 863 
 864     if (op_msg == NULL) {
 865         return -EINVAL;
 866     }
 867 
 868     if (expect_reply) {
 869         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 870     } else {
 871         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 872         goto done;
 873     }
 874 
 875     if (rc < 0) {
 876         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 877         rc = -ECOMM;
 878         goto done;
 879 
 880     } else if(op_reply == NULL) {
 881         rc = -ENOMSG;
 882         goto done;
 883     }
 884 
 885     rc = pcmk_ok;
 886     crm_trace("%s op reply received", op);
 887     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
 888         rc = -ENOMSG;
 889         goto done;
 890     }
 891 
 892     crm_log_xml_trace(op_reply, "Reply");
 893 
 894     if (output_data) {
 895         *output_data = op_reply;
 896         op_reply = NULL;        /* Prevent subsequent free */
 897     }
 898 
 899   done:
 900     if (lrmd_api_is_connected(lrmd) == FALSE) {
 901         crm_err("Executor disconnected");
 902     }
 903 
 904     free_xml(op_msg);
 905     free_xml(op_reply);
 906     return rc;
 907 }
 908 
 909 static int
 910 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 911 {
 912     int rc;
 913     lrmd_private_t *native = lrmd->lrmd_private;
 914     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 915 
 916     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 917     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 918                            (native->type == pcmk__client_ipc));
 919     free_xml(data);
 920 
 921     return rc < 0 ? rc : pcmk_ok;
 922 }
 923 
 924 int
 925 remote_proxy_check(lrmd_t * lrmd, GHashTable *hash)
     /*  */
 926 {
 927     int rc;
 928     const char *value;
 929     lrmd_private_t *native = lrmd->lrmd_private;
 930     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 931 
 932     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 933 
 934     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
 935     crm_xml_add(data, F_LRMD_WATCHDOG, value);
 936 
 937     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
 938                            (native->type == pcmk__client_ipc));
 939     free_xml(data);
 940 
 941     return rc < 0 ? rc : pcmk_ok;
 942 }
 943 
 944 static int
 945 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
 946 {
 947     int rc = pcmk_ok;
 948     lrmd_private_t *native = lrmd->lrmd_private;
 949     xmlNode *reply = NULL;
 950     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 951 
 952     crm_xml_add(hello, F_TYPE, T_LRMD);
 953     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
 954     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
 955     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 956 
 957     /* advertise that we are a proxy provider */
 958     if (native->proxy_callback) {
 959         crm_xml_add(hello, F_LRMD_IS_IPC_PROVIDER, "true");
 960     }
 961 
 962     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 963 
 964     if (rc < 0) {
 965         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
 966         rc = -ECOMM;
 967     } else if (reply == NULL) {
 968         crm_err("Did not receive registration reply");
 969         rc = -EPROTO;
 970     } else {
 971         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
 972         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
 973         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 974 
 975         crm_element_value_int(reply, F_LRMD_RC, &rc);
 976 
 977         if (rc == -EPROTO) {
 978             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
 979                 LRMD_PROTOCOL_VERSION, version);
 980             crm_log_xml_err(reply, "Protocol Error");
 981 
 982         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
 983             crm_err("Invalid registration message: %s", msg_type);
 984             crm_log_xml_err(reply, "Bad reply");
 985             rc = -EPROTO;
 986         } else if (tmp_ticket == NULL) {
 987             crm_err("No registration token provided");
 988             crm_log_xml_err(reply, "Bad reply");
 989             rc = -EPROTO;
 990         } else {
 991             crm_trace("Obtained registration token: %s", tmp_ticket);
 992             native->token = strdup(tmp_ticket);
 993             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
 994             rc = pcmk_ok;
 995         }
 996     }
 997 
 998     free_xml(reply);
 999     free_xml(hello);
1000 
1001     if (rc != pcmk_ok) {
1002         lrmd_api_disconnect(lrmd);
1003     }
1004     return rc;
1005 }
1006 
1007 static int
1008 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1009 {
1010     int rc = pcmk_ok;
1011     lrmd_private_t *native = lrmd->lrmd_private;
1012 
1013     struct ipc_client_callbacks lrmd_callbacks = {
1014         .dispatch = lrmd_ipc_dispatch,
1015         .destroy = lrmd_ipc_connection_destroy
1016     };
1017 
1018     crm_info("Connecting to executor");
1019 
1020     if (fd) {
1021         /* No mainloop */
1022         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1023         if (native->ipc && crm_ipc_connect(native->ipc)) {
1024             *fd = crm_ipc_get_fd(native->ipc);
1025         } else if (native->ipc) {
1026             crm_perror(LOG_ERR, "Connection to executor failed");
1027             rc = -ENOTCONN;
1028         }
1029     } else {
1030         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1031         native->ipc = mainloop_get_ipc_client(native->source);
1032     }
1033 
1034     if (native->ipc == NULL) {
1035         crm_debug("Could not connect to the executor API");
1036         rc = -ENOTCONN;
1037     }
1038 
1039     return rc;
1040 }
1041 
1042 #ifdef HAVE_GNUTLS_GNUTLS_H
1043 static void
1044 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1045 {
1046     dest->data = gnutls_malloc(source->size);
1047     CRM_ASSERT(dest->data);
1048     memcpy(dest->data, source->data, source->size);
1049     dest->size = source->size;
1050 }
1051 
1052 static void
1053 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1054 {
1055     gnutls_free(datum->data);
1056     datum->data = NULL;
1057     datum->size = 0;
1058 }
1059 
1060 #define KEY_READ_LEN 256
1061 
1062 static int
1063 set_key(gnutls_datum_t * key, const char *location)
     /*  */
1064 {
1065     FILE *stream;
1066     size_t buf_len = KEY_READ_LEN;
1067     static gnutls_datum_t key_cache = { 0, };
1068     static time_t key_cache_updated = 0;
1069 
1070     if (location == NULL) {
1071         return -1;
1072     }
1073 
1074     if (key_cache.data != NULL) {
1075         if ((time(NULL) - key_cache_updated) < 60) {
1076             copy_gnutls_datum(key, &key_cache);
1077             crm_debug("Using cached Pacemaker Remote key");
1078             return 0;
1079         } else {
1080             clear_gnutls_datum(&key_cache);
1081             key_cache_updated = 0;
1082             crm_debug("Cleared Pacemaker Remote key cache");
1083         }
1084     }
1085 
1086     stream = fopen(location, "r");
1087     if (!stream) {
1088         return -1;
1089     }
1090 
1091     key->data = gnutls_malloc(buf_len);
1092     key->size = 0;
1093     while (!feof(stream)) {
1094         int next = fgetc(stream);
1095 
1096         if (next == EOF) {
1097             if (!feof(stream)) {
1098                 crm_err("Error reading Pacemaker Remote key; copy in memory may be corrupted");
1099             }
1100             break;
1101         }
1102         if (key->size == buf_len) {
1103             buf_len = key->size + KEY_READ_LEN;
1104             key->data = gnutls_realloc(key->data, buf_len);
1105             CRM_ASSERT(key->data);
1106         }
1107         key->data[key->size++] = (unsigned char) next;
1108     }
1109     fclose(stream);
1110 
1111     if (key->size == 0) {
1112         clear_gnutls_datum(key);
1113         return -1;
1114     }
1115 
1116     if (key_cache.data == NULL) {
1117         copy_gnutls_datum(&key_cache, key);
1118         key_cache_updated = time(NULL);
1119         crm_debug("Cached Pacemaker Remote key");
1120     }
1121 
1122     return 0;
1123 }
1124 
1125 int
1126 lrmd_tls_set_key(gnutls_datum_t * key)
     /*  */
1127 {
1128     const char *specific_location = getenv("PCMK_authkey_location");
1129 
1130     if (set_key(key, specific_location) == 0) {
1131         crm_debug("Using custom authkey location %s", specific_location);
1132         return pcmk_ok;
1133 
1134     } else if (specific_location) {
1135         crm_err("No valid Pacemaker Remote key found at %s, trying default location", specific_location);
1136     }
1137 
1138     if ((set_key(key, DEFAULT_REMOTE_KEY_LOCATION) != 0)
1139         && (set_key(key, ALT_REMOTE_KEY_LOCATION) != 0)) {
1140         crm_err("No valid Pacemaker Remote key found at %s", DEFAULT_REMOTE_KEY_LOCATION);
1141         return -ENOKEY;
1142     }
1143 
1144     return pcmk_ok;
1145 }
1146 
1147 static void
1148 lrmd_gnutls_global_init(void)
     /*  */
1149 {
1150     static int gnutls_init = 0;
1151 
1152     if (!gnutls_init) {
1153         crm_gnutls_global_init();
1154     }
1155     gnutls_init = 1;
1156 }
1157 #endif
1158 
1159 static void
1160 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1161 {
1162     lrmd_private_t *native = lrmd->lrmd_private;
1163 
1164     if (native->callback) {
1165         lrmd_event_data_t event = { 0, };
1166         event.type = lrmd_event_connect;
1167         event.remote_nodename = native->remote_nodename;
1168         event.connection_rc = rc;
1169         native->callback(&event);
1170     }
1171 }
1172 
1173 #ifdef HAVE_GNUTLS_GNUTLS_H
1174 static inline int
1175 lrmd__tls_client_handshake(pcmk__remote_t *remote)
     /*  */
1176 {
1177     return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
1178 }
1179 
1180 static void
1181 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1182 {
1183     lrmd_t *lrmd = userdata;
1184     lrmd_private_t *native = lrmd->lrmd_private;
1185     char *name;
1186     static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
1187         .dispatch = lrmd_tls_dispatch,
1188         .destroy = lrmd_tls_connection_destroy,
1189     };
1190     gnutls_datum_t psk_key = { NULL, 0 };
1191 
1192     native->async_timer = 0;
1193 
1194     if (rc != pcmk_rc_ok) {
1195         lrmd_tls_connection_destroy(lrmd);
1196         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1197                  CRM_XS " rc=%d",
1198                  native->server, native->port, pcmk_rc_str(rc), rc);
1199         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1200         return;
1201     }
1202 
1203     /* The TCP connection was successful, so establish the TLS connection.
1204      * @TODO make this async to avoid blocking code in client
1205      */
1206 
1207     native->sock = sock;
1208 
1209     rc = lrmd_tls_set_key(&psk_key);
1210     if (rc != 0) {
1211         crm_warn("Could not set key for Pacemaker Remote at %s:%d " CRM_XS " rc=%d",
1212                  native->server, native->port, rc);
1213         lrmd_tls_connection_destroy(lrmd);
1214         report_async_connection_result(lrmd, rc);
1215         return;
1216     }
1217 
1218     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1219     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1220     gnutls_free(psk_key.data);
1221 
1222     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
1223                                                         GNUTLS_CRD_PSK,
1224                                                         native->psk_cred_c);
1225     if (native->remote->tls_session == NULL) {
1226         lrmd_tls_connection_destroy(lrmd);
1227         report_async_connection_result(lrmd, -EPROTO);
1228         return;
1229     }
1230 
1231     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1232         crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
1233                  native->server, native->port);
1234         gnutls_deinit(*native->remote->tls_session);
1235         gnutls_free(native->remote->tls_session);
1236         native->remote->tls_session = NULL;
1237         lrmd_tls_connection_destroy(lrmd);
1238         report_async_connection_result(lrmd, -EKEYREJECTED);
1239         return;
1240     }
1241 
1242     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1243              native->server, native->port);
1244 
1245     name = crm_strdup_printf("pacemaker-remote-%s:%d",
1246                              native->server, native->port);
1247 
1248     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
1249     native->source =
1250         mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
1251 
1252     rc = lrmd_handshake(lrmd, name);
1253     free(name);
1254 
1255     report_async_connection_result(lrmd, rc);
1256     return;
1257 }
1258 
1259 static int
1260 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1261 {
1262     int rc;
1263     int timer_id = 0;
1264     lrmd_private_t *native = lrmd->lrmd_private;
1265 
1266     lrmd_gnutls_global_init();
1267     native->sock = -1;
1268     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1269                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1270     if (rc != pcmk_rc_ok) {
1271         crm_warn("Pacemaker Remote connection to %s:%s failed: %s "
1272                  CRM_XS " rc=%d",
1273                  native->server, native->port, pcmk_rc_str(rc), rc);
1274         return -1;
1275     }
1276     native->async_timer = timer_id;
1277     return pcmk_ok;
1278 }
1279 
1280 static int
1281 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1282 {
1283     static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
1284         .dispatch = lrmd_tls_dispatch,
1285         .destroy = lrmd_tls_connection_destroy,
1286     };
1287     int rc;
1288 
1289     lrmd_private_t *native = lrmd->lrmd_private;
1290     gnutls_datum_t psk_key = { NULL, 0 };
1291 
1292     lrmd_gnutls_global_init();
1293 
1294     native->sock = -1;
1295     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1296                               &(native->sock), NULL, NULL);
1297     if (rc != pcmk_rc_ok) {
1298         crm_warn("Pacemaker Remote connection to %s:%s failed: %s "
1299                  CRM_XS " rc=%d",
1300                  native->server, native->port, pcmk_rc_str(rc), rc);
1301         lrmd_tls_connection_destroy(lrmd);
1302         return -ENOTCONN;
1303     }
1304 
1305     rc = lrmd_tls_set_key(&psk_key);
1306     if (rc < 0) {
1307         lrmd_tls_connection_destroy(lrmd);
1308         return rc;
1309     }
1310 
1311     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1312     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1313     gnutls_free(psk_key.data);
1314 
1315     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
1316                                                         GNUTLS_CRD_PSK,
1317                                                         native->psk_cred_c);
1318     if (native->remote->tls_session == NULL) {
1319         lrmd_tls_connection_destroy(lrmd);
1320         return -EPROTO;
1321     }
1322 
1323     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1324         crm_err("Session creation for %s:%d failed", native->server, native->port);
1325         gnutls_deinit(*native->remote->tls_session);
1326         gnutls_free(native->remote->tls_session);
1327         native->remote->tls_session = NULL;
1328         lrmd_tls_connection_destroy(lrmd);
1329         return -EKEYREJECTED;
1330     }
1331 
1332     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1333              native->port);
1334 
1335     if (fd) {
1336         *fd = native->sock;
1337     } else {
1338         char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1339                                        native->server, native->port);
1340 
1341         native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
1342         native->source =
1343             mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
1344         free(name);
1345     }
1346     return pcmk_ok;
1347 }
1348 #endif
1349 
1350 static int
1351 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1352 {
1353     int rc = -ENOTCONN;
1354     lrmd_private_t *native = lrmd->lrmd_private;
1355 
1356     switch (native->type) {
1357         case pcmk__client_ipc:
1358             rc = lrmd_ipc_connect(lrmd, fd);
1359             break;
1360 #ifdef HAVE_GNUTLS_GNUTLS_H
1361         case pcmk__client_tls:
1362             rc = lrmd_tls_connect(lrmd, fd);
1363             break;
1364 #endif
1365         default:
1366             crm_err("Unsupported connection type: %d", native->type);
1367     }
1368 
1369     if (rc == pcmk_ok) {
1370         rc = lrmd_handshake(lrmd, name);
1371     }
1372 
1373     return rc;
1374 }
1375 
1376 static int
1377 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1378 {
1379     int rc = 0;
1380     lrmd_private_t *native = lrmd->lrmd_private;
1381 
1382     CRM_CHECK(native && native->callback, return -1);
1383 
1384     switch (native->type) {
1385         case pcmk__client_ipc:
1386             /* fake async connection with ipc.  it should be fast
1387              * enough that we gain very little from async */
1388             rc = lrmd_api_connect(lrmd, name, NULL);
1389             if (!rc) {
1390                 report_async_connection_result(lrmd, rc);
1391             }
1392             break;
1393 #ifdef HAVE_GNUTLS_GNUTLS_H
1394         case pcmk__client_tls:
1395             rc = lrmd_tls_connect_async(lrmd, timeout);
1396             if (rc) {
1397                 /* connection failed, report rc now */
1398                 report_async_connection_result(lrmd, rc);
1399             }
1400             break;
1401 #endif
1402         default:
1403             crm_err("Unsupported connection type: %d", native->type);
1404     }
1405 
1406     return rc;
1407 }
1408 
1409 static void
1410 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1411 {
1412     lrmd_private_t *native = lrmd->lrmd_private;
1413 
1414     if (native->source != NULL) {
1415         /* Attached to mainloop */
1416         mainloop_del_ipc_client(native->source);
1417         native->source = NULL;
1418         native->ipc = NULL;
1419 
1420     } else if (native->ipc) {
1421         /* Not attached to mainloop */
1422         crm_ipc_t *ipc = native->ipc;
1423 
1424         native->ipc = NULL;
1425         crm_ipc_close(ipc);
1426         crm_ipc_destroy(ipc);
1427     }
1428 }
1429 
1430 #ifdef HAVE_GNUTLS_GNUTLS_H
1431 static void
1432 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1433 {
1434     lrmd_private_t *native = lrmd->lrmd_private;
1435 
1436     if (native->remote->tls_session) {
1437         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1438         gnutls_deinit(*native->remote->tls_session);
1439         gnutls_free(native->remote->tls_session);
1440         native->remote->tls_session = 0;
1441     }
1442 
1443     if (native->async_timer) {
1444         g_source_remove(native->async_timer);
1445         native->async_timer = 0;
1446     }
1447 
1448     if (native->source != NULL) {
1449         /* Attached to mainloop */
1450         mainloop_del_ipc_client(native->source);
1451         native->source = NULL;
1452 
1453     } else if (native->sock) {
1454         close(native->sock);
1455         native->sock = 0;
1456     }
1457 
1458     if (native->pending_notify) {
1459         g_list_free_full(native->pending_notify, lrmd_free_xml);
1460         native->pending_notify = NULL;
1461     }
1462 }
1463 #endif
1464 
1465 static int
1466 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1467 {
1468     lrmd_private_t *native = lrmd->lrmd_private;
1469 
1470     crm_info("Disconnecting %s %s executor connection",
1471              pcmk__client_type_str(native->type),
1472              (native->remote_nodename? native->remote_nodename : "local"));
1473     switch (native->type) {
1474         case pcmk__client_ipc:
1475             lrmd_ipc_disconnect(lrmd);
1476             break;
1477 #ifdef HAVE_GNUTLS_GNUTLS_H
1478         case pcmk__client_tls:
1479             lrmd_tls_disconnect(lrmd);
1480             break;
1481 #endif
1482         default:
1483             crm_err("Unsupported connection type: %d", native->type);
1484     }
1485 
1486     free(native->token);
1487     native->token = NULL;
1488 
1489     free(native->peer_version);
1490     native->peer_version = NULL;
1491     return 0;
1492 }
1493 
1494 static int
1495 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1496                       const char *rsc_id,
1497                       const char *class,
1498                       const char *provider, const char *type, enum lrmd_call_options options)
1499 {
1500     int rc = pcmk_ok;
1501     xmlNode *data = NULL;
1502 
1503     if (!class || !type || !rsc_id) {
1504         return -EINVAL;
1505     }
1506     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1507         && (provider == NULL)) {
1508         return -EINVAL;
1509     }
1510 
1511     data = create_xml_node(NULL, F_LRMD_RSC);
1512 
1513     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1514     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1515     crm_xml_add(data, F_LRMD_CLASS, class);
1516     crm_xml_add(data, F_LRMD_PROVIDER, provider);
1517     crm_xml_add(data, F_LRMD_TYPE, type);
1518     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1519     free_xml(data);
1520 
1521     return rc;
1522 }
1523 
1524 static int
1525 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1526 {
1527     int rc = pcmk_ok;
1528     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1529 
1530     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1531     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1532     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1533     free_xml(data);
1534 
1535     return rc;
1536 }
1537 
1538 lrmd_rsc_info_t *
1539 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1540                   const char *provider, const char *type)
1541 {
1542     lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
1543 
1544     CRM_ASSERT(rsc_info);
1545     if (rsc_id) {
1546         rsc_info->id = strdup(rsc_id);
1547         CRM_ASSERT(rsc_info->id);
1548     }
1549     if (standard) {
1550         rsc_info->standard = strdup(standard);
1551         CRM_ASSERT(rsc_info->standard);
1552     }
1553     if (provider) {
1554         rsc_info->provider = strdup(provider);
1555         CRM_ASSERT(rsc_info->provider);
1556     }
1557     if (type) {
1558         rsc_info->type = strdup(type);
1559         CRM_ASSERT(rsc_info->type);
1560     }
1561     return rsc_info;
1562 }
1563 
1564 lrmd_rsc_info_t *
1565 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1566 {
1567     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1568                              rsc_info->provider, rsc_info->type);
1569 }
1570 
1571 void
1572 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1573 {
1574     if (!rsc_info) {
1575         return;
1576     }
1577     free(rsc_info->id);
1578     free(rsc_info->type);
1579     free(rsc_info->standard);
1580     free(rsc_info->provider);
1581     free(rsc_info);
1582 }
1583 
1584 static lrmd_rsc_info_t *
1585 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1586 {
1587     lrmd_rsc_info_t *rsc_info = NULL;
1588     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1589     xmlNode *output = NULL;
1590     const char *class = NULL;
1591     const char *provider = NULL;
1592     const char *type = NULL;
1593 
1594     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1595     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1596     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1597     free_xml(data);
1598 
1599     if (!output) {
1600         return NULL;
1601     }
1602 
1603     class = crm_element_value(output, F_LRMD_CLASS);
1604     provider = crm_element_value(output, F_LRMD_PROVIDER);
1605     type = crm_element_value(output, F_LRMD_TYPE);
1606 
1607     if (!class || !type) {
1608         free_xml(output);
1609         return NULL;
1610     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1611                && !provider) {
1612         free_xml(output);
1613         return NULL;
1614     }
1615 
1616     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1617     free_xml(output);
1618     return rsc_info;
1619 }
1620 
1621 void
1622 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1623 {
1624     if (op_info) {
1625         free(op_info->rsc_id);
1626         free(op_info->action);
1627         free(op_info->interval_ms_s);
1628         free(op_info->timeout_ms_s);
1629         free(op_info);
1630     }
1631 }
1632 
1633 static int
1634 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1635                            enum lrmd_call_options options, GList **output)
1636 {
1637     xmlNode *data = NULL;
1638     xmlNode *output_xml = NULL;
1639     int rc = pcmk_ok;
1640 
1641     if (output == NULL) {
1642         return -EINVAL;
1643     }
1644     *output = NULL;
1645 
1646     // Send request
1647     if (rsc_id) {
1648         data = create_xml_node(NULL, F_LRMD_RSC);
1649         crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1650         crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1651     }
1652     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1653                            timeout_ms, options, TRUE);
1654     if (data) {
1655         free_xml(data);
1656     }
1657 
1658     // Process reply
1659     if ((rc != pcmk_ok) || (output_xml == NULL)) {
1660         return rc;
1661     }
1662     for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
1663          rsc_xml != NULL; rsc_xml = crm_next_same_xml(rsc_xml)) {
1664 
1665         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1666         if (rsc_id == NULL) {
1667             crm_err("Could not parse recurring operation information from executor");
1668             continue;
1669         }
1670         for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
1671              op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
1672 
1673             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
1674 
1675             CRM_CHECK(op_info != NULL, break);
1676             op_info->rsc_id = strdup(rsc_id);
1677             op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
1678             op_info->interval_ms_s = crm_element_value_copy(op_xml,
1679                                                             F_LRMD_RSC_INTERVAL);
1680             op_info->timeout_ms_s = crm_element_value_copy(op_xml,
1681                                                            F_LRMD_TIMEOUT);
1682             *output = g_list_prepend(*output, op_info);
1683         }
1684     }
1685     free_xml(output_xml);
1686     return rc;
1687 }
1688 
1689 
1690 static void
1691 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
1692 {
1693     lrmd_private_t *native = lrmd->lrmd_private;
1694 
1695     native->callback = callback;
1696 }
1697 
1698 void
1699 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
1700 {
1701     lrmd_private_t *native = lrmd->lrmd_private;
1702 
1703     native->proxy_callback = callback;
1704     native->proxy_callback_userdata = userdata;
1705 }
1706 
1707 void
1708 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
1709 {
1710     lrmd_private_t *native = lrmd->lrmd_private;
1711 
1712     if (native->proxy_callback) {
1713         crm_log_xml_trace(msg, "PROXY_INBOUND");
1714         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
1715     }
1716 }
1717 
1718 int
1719 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
1720 {
1721     if (lrmd == NULL) {
1722         return -ENOTCONN;
1723     }
1724     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
1725 
1726     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
1727     return lrmd_send_xml_no_reply(lrmd, msg);
1728 }
1729 
1730 static int
1731 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
1732 {
1733     int rc = pcmk_ok;
1734     stonith_t *stonith_api = stonith_api_new();
1735 
1736     if (stonith_api == NULL) {
1737         crm_err("Could not get fence agent meta-data: API memory allocation failed");
1738         return -ENOMEM;
1739     }
1740 
1741     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
1742                                      provider, output, 0);
1743     if ((rc == pcmk_ok) && (*output == NULL)) {
1744         rc = -EIO;
1745     }
1746     stonith_api->cmds->free(stonith_api);
1747     return rc;
1748 }
1749 
1750 static int
1751 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
1752                       const char *type, char **output,
1753                       enum lrmd_call_options options)
1754 {
1755     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
1756                                            output, options, NULL);
1757 }
1758 
1759 static int
1760 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
1761                              const char *provider, const char *type,
1762                              char **output, enum lrmd_call_options options,
1763                              lrmd_key_value_t *params)
1764 {
1765     svc_action_t *action = NULL;
1766     GHashTable *params_table = NULL;
1767 
1768     if (!standard || !type) {
1769         lrmd_key_value_freeall(params);
1770         return -EINVAL;
1771     }
1772 
1773     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
1774         lrmd_key_value_freeall(params);
1775         return stonith_get_metadata(provider, type, output);
1776     }
1777 
1778     params_table = crm_str_table_new();
1779     for (const lrmd_key_value_t *param = params; param; param = param->next) {
1780         g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
1781     }
1782     action = resources_action_create(type, standard, provider, type,
1783                                      CRMD_ACTION_METADATA, 0,
1784                                      CRMD_METADATA_CALL_TIMEOUT, params_table,
1785                                      0);
1786     lrmd_key_value_freeall(params);
1787 
1788     if (action == NULL) {
1789         crm_err("Unable to retrieve meta-data for %s:%s:%s",
1790                 standard, provider, type);
1791         return -EINVAL;
1792     }
1793 
1794     if (!services_action_sync(action)) {
1795         crm_err("Failed to retrieve meta-data for %s:%s:%s",
1796                 standard, provider, type);
1797         services_action_free(action);
1798         return -EIO;
1799     }
1800 
1801     if (!action->stdout_data) {
1802         crm_err("Failed to receive meta-data for %s:%s:%s",
1803                 standard, provider, type);
1804         services_action_free(action);
1805         return -EIO;
1806     }
1807 
1808     *output = strdup(action->stdout_data);
1809     services_action_free(action);
1810 
1811     return pcmk_ok;
1812 }
1813 
1814 static int
1815 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
1816               const char *userdata, guint interval_ms,
1817               int timeout,      /* ms */
1818               int start_delay,  /* ms */
1819               enum lrmd_call_options options, lrmd_key_value_t * params)
1820 {
1821     int rc = pcmk_ok;
1822     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1823     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
1824     lrmd_key_value_t *tmp = NULL;
1825 
1826     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1827     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1828     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
1829     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
1830     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
1831     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
1832     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
1833 
1834     for (tmp = params; tmp; tmp = tmp->next) {
1835         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
1836     }
1837 
1838     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
1839     free_xml(data);
1840 
1841     lrmd_key_value_freeall(params);
1842     return rc;
1843 }
1844 
1845 /* timeout is in ms */
1846 static int
1847 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
1848                     int timeout, lrmd_key_value_t *params)
1849 {
1850     int rc = pcmk_ok;
1851     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
1852     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
1853     lrmd_key_value_t *tmp = NULL;
1854 
1855     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1856     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
1857     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
1858     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
1859 
1860     for (tmp = params; tmp; tmp = tmp->next) {
1861         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
1862     }
1863 
1864     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
1865                            lrmd_opt_notify_orig_only, TRUE);
1866     free_xml(data);
1867 
1868     lrmd_key_value_freeall(params);
1869     return rc;
1870 }
1871 
1872 static int
1873 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
1874                 guint interval_ms)
1875 {
1876     int rc = pcmk_ok;
1877     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1878 
1879     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1880     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
1881     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1882     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
1883     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
1884     free_xml(data);
1885     return rc;
1886 }
1887 
1888 static int
1889 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
1890 {
1891     int rc = 0;
1892     stonith_t *stonith_api = stonith_api_new();
1893     stonith_key_value_t *stonith_resources = NULL;
1894     stonith_key_value_t *dIter = NULL;
1895 
1896     if (stonith_api == NULL) {
1897         crm_err("Could not list fence agents: API memory allocation failed");
1898         return -ENOMEM;
1899     }
1900     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
1901                                    &stonith_resources, 0);
1902     stonith_api->cmds->free(stonith_api);
1903 
1904     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
1905         rc++;
1906         if (resources) {
1907             *resources = lrmd_list_add(*resources, dIter->value);
1908         }
1909     }
1910 
1911     stonith_key_value_freeall(stonith_resources, 1, 0);
1912     return rc;
1913 }
1914 
1915 static int
1916 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
1917                      const char *provider)
1918 {
1919     int rc = 0;
1920     int stonith_count = 0; // Initially, whether to include stonith devices
1921 
1922     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
1923         stonith_count = 1;
1924 
1925     } else {
1926         GListPtr gIter = NULL;
1927         GList *agents = resources_list_agents(class, provider);
1928 
1929         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
1930             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
1931             rc++;
1932         }
1933         g_list_free_full(agents, free);
1934 
1935         if (!class) {
1936             stonith_count = 1;
1937         }
1938     }
1939 
1940     if (stonith_count) {
1941         // Now, if stonith devices are included, how many there are
1942         stonith_count = list_stonith_agents(resources);
1943         if (stonith_count > 0) {
1944             rc += stonith_count;
1945         }
1946     }
1947     if (rc == 0) {
1948         crm_notice("No agents found for class %s", class);
1949         rc = -EPROTONOSUPPORT;
1950     }
1951     return rc;
1952 }
1953 
1954 static int
1955 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
1956 {
1957     int found = 0;
1958     GList *agents = NULL;
1959     GListPtr gIter2 = NULL;
1960 
1961     agents = resources_list_agents(class, provider);
1962     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
1963         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
1964             found = 1;
1965         }
1966     }
1967     g_list_free_full(agents, free);
1968 
1969     return found;
1970 }
1971 
1972 static int
1973 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
1974 {
1975     int rc = pcmk_ok;
1976     char *provider = NULL;
1977     GList *ocf_providers = NULL;
1978     GListPtr gIter = NULL;
1979 
1980     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
1981 
1982     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
1983         provider = gIter->data;
1984         if (!agent || does_provider_have_agent(agent, provider,
1985                                                PCMK_RESOURCE_CLASS_OCF)) {
1986             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
1987             rc++;
1988         }
1989     }
1990 
1991     g_list_free_full(ocf_providers, free);
1992     return rc;
1993 }
1994 
1995 static int
1996 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
1997 {
1998     int rc = 0;
1999     GList *standards = NULL;
2000     GListPtr gIter = NULL;
2001 
2002     standards = resources_list_standards();
2003 
2004     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2005         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2006         rc++;
2007     }
2008 
2009     if (list_stonith_agents(NULL) > 0) {
2010         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2011         rc++;
2012     }
2013 
2014     g_list_free_full(standards, free);
2015     return rc;
2016 }
2017 
2018 lrmd_t *
2019 lrmd_api_new(void)
     /*  */
2020 {
2021     lrmd_t *new_lrmd = NULL;
2022     lrmd_private_t *pvt = NULL;
2023 
2024     new_lrmd = calloc(1, sizeof(lrmd_t));
2025     pvt = calloc(1, sizeof(lrmd_private_t));
2026     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2027     new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2028 
2029     pvt->type = pcmk__client_ipc;
2030     new_lrmd->lrmd_private = pvt;
2031 
2032     new_lrmd->cmds->connect = lrmd_api_connect;
2033     new_lrmd->cmds->connect_async = lrmd_api_connect_async;
2034     new_lrmd->cmds->is_connected = lrmd_api_is_connected;
2035     new_lrmd->cmds->poke_connection = lrmd_api_poke_connection;
2036     new_lrmd->cmds->disconnect = lrmd_api_disconnect;
2037     new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
2038     new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2039     new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2040     new_lrmd->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2041     new_lrmd->cmds->set_callback = lrmd_api_set_callback;
2042     new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
2043     new_lrmd->cmds->exec = lrmd_api_exec;
2044     new_lrmd->cmds->cancel = lrmd_api_cancel;
2045     new_lrmd->cmds->list_agents = lrmd_api_list_agents;
2046     new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2047     new_lrmd->cmds->list_standards = lrmd_api_list_standards;
2048     new_lrmd->cmds->exec_alert = lrmd_api_exec_alert;
2049     new_lrmd->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2050 
2051     return new_lrmd;
2052 }
2053 
2054 lrmd_t *
2055 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2056 {
2057 #ifdef HAVE_GNUTLS_GNUTLS_H
2058     lrmd_t *new_lrmd = lrmd_api_new();
2059     lrmd_private_t *native = new_lrmd->lrmd_private;
2060 
2061     if (!nodename && !server) {
2062         lrmd_api_delete(new_lrmd);
2063         return NULL;
2064     }
2065 
2066     native->type = pcmk__client_tls;
2067     native->remote_nodename = nodename ? strdup(nodename) : strdup(server);
2068     native->server = server ? strdup(server) : strdup(nodename);
2069     native->port = port;
2070     if (native->port == 0) {
2071         native->port = crm_default_remote_port();
2072     }
2073 
2074     return new_lrmd;
2075 #else
2076     crm_err("Cannot communicate with Pacemaker Remote because GnuTLS is not enabled for this build");
2077     return NULL;
2078 #endif
2079 }
2080 
2081 void
2082 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2083 {
2084     if (!lrmd) {
2085         return;
2086     }
2087     lrmd->cmds->disconnect(lrmd);       /* no-op if already disconnected */
2088     free(lrmd->cmds);
2089     if (lrmd->lrmd_private) {
2090         lrmd_private_t *native = lrmd->lrmd_private;
2091 
2092 #ifdef HAVE_GNUTLS_GNUTLS_H
2093         free(native->server);
2094 #endif
2095         free(native->remote_nodename);
2096         free(native->remote);
2097         free(native->token);
2098         free(native->peer_version);
2099     }
2100 
2101     free(lrmd->lrmd_private);
2102     free(lrmd);
2103 }
/* */
root/lib/lrmd/lrmd_client.c

DEFINITIONS
