corosync  2.3.2
totemudpu.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2005 MontaVista Software, Inc.
3  * Copyright (c) 2006-2012 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8 
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <assert.h>
39 #include <sys/mman.h>
40 #include <sys/types.h>
41 #include <sys/stat.h>
42 #include <sys/socket.h>
43 #include <netdb.h>
44 #include <sys/un.h>
45 #include <sys/ioctl.h>
46 #include <sys/param.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <sched.h>
55 #include <time.h>
56 #include <sys/time.h>
57 #include <sys/poll.h>
58 #include <sys/uio.h>
59 #include <limits.h>
60 
61 #include <qb/qbdefs.h>
62 #include <qb/qbloop.h>
63 
64 #include <corosync/sq.h>
65 #include <corosync/list.h>
66 #include <corosync/swab.h>
67 #define LOGSYS_UTILS_ONLY 1
68 #include <corosync/logsys.h>
69 #include "totemudpu.h"
70 
71 #include "util.h"
72 #include "totemcrypto.h"
73 
74 #include <nss.h>
75 #include <pk11pub.h>
76 #include <pkcs11.h>
77 #include <prerror.h>
78 
79 #ifndef MSG_NOSIGNAL
80 #define MSG_NOSIGNAL 0
81 #endif
82 
83 #define MCAST_SOCKET_BUFFER_SIZE (TRANSMITS_ALLOWED * FRAME_SIZE_MAX)
84 #define NETIF_STATE_REPORT_UP 1
85 #define NETIF_STATE_REPORT_DOWN 2
86 
87 #define BIND_STATE_UNBOUND 0
88 #define BIND_STATE_REGULAR 1
89 #define BIND_STATE_LOOPBACK 2
90 
92  struct list_head list;
94  int fd;
95 };
96 
99 
101 
103 
105 
107 
108  void *context;
109 
111  void *context,
112  const void *msg,
113  unsigned int msg_len);
114 
116  void *context,
117  const struct totem_ip_address *iface_address);
118 
120 
121  /*
122  * Function and data used to log messages
123  */
125 
127 
129 
131 
133 
135 
137  int level,
138  int subsys,
139  const char *function,
140  const char *file,
141  int line,
142  const char *format,
143  ...)__attribute__((format(printf, 6, 7)));
144 
145  void *udpu_context;
146 
148 
149  struct iovec totemudpu_iov_recv;
150 
152 
154 
156 
158 
160 
162 
163  struct timeval stats_tv_start;
164 
166 
167  int firstrun;
168 
169  qb_loop_timer_handle timer_netif_check_timeout;
170 
171  unsigned int my_memb_entries;
172 
174 
176 
178 
180 };
181 
182 struct work_item {
183  const void *msg;
184  unsigned int msg_len;
186 };
187 
188 static int totemudpu_build_sockets (
189  struct totemudpu_instance *instance,
190  struct totem_ip_address *bindnet_address,
191  struct totem_ip_address *bound_to);
192 
193 static int totemudpu_create_sending_socket(
194  void *udpu_context,
195  const struct totem_ip_address *member);
196 
198  void *udpu_context);
199 
200 static struct totem_ip_address localhost;
201 
202 static void totemudpu_instance_initialize (struct totemudpu_instance *instance)
203 {
204  memset (instance, 0, sizeof (struct totemudpu_instance));
205 
207 
208  instance->totemudpu_iov_recv.iov_base = instance->iov_buffer;
209 
210  instance->totemudpu_iov_recv.iov_len = FRAME_SIZE_MAX; //sizeof (instance->iov_buffer);
211 
212  /*
213  * There is always atleast 1 processor
214  */
215  instance->my_memb_entries = 1;
216 
217  list_init (&instance->member_list);
218 }
219 
220 #define log_printf(level, format, args...) \
221 do { \
222  instance->totemudpu_log_printf ( \
223  level, instance->totemudpu_subsys_id, \
224  __FUNCTION__, __FILE__, __LINE__, \
225  (const char *)format, ##args); \
226 } while (0);
227 #define LOGSYS_PERROR(err_num, level, fmt, args...) \
228 do { \
229  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
230  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
231  instance->totemudpu_log_printf ( \
232  level, instance->totemudpu_subsys_id, \
233  __FUNCTION__, __FILE__, __LINE__, \
234  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
235  } while(0)
236 
238  void *udpu_context,
239  const char *cipher_type,
240  const char *hash_type)
241 {
242 
243  return (0);
244 }
245 
246 
247 static inline void ucast_sendmsg (
248  struct totemudpu_instance *instance,
249  struct totem_ip_address *system_to,
250  const void *msg,
251  unsigned int msg_len)
252 {
253  struct msghdr msg_ucast;
254  int res = 0;
255  size_t buf_out_len;
256  unsigned char buf_out[FRAME_SIZE_MAX];
257  struct sockaddr_storage sockaddr;
258  struct iovec iovec;
259  int addrlen;
260 
261  /*
262  * Encrypt and digest the message
263  */
265  instance->crypto_inst,
266  (const unsigned char *)msg,
267  msg_len,
268  buf_out,
269  &buf_out_len) != 0) {
270  log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
271  return;
272  }
273 
274  iovec.iov_base = (void *)buf_out;
275  iovec.iov_len = buf_out_len;
276 
277  /*
278  * Build unicast message
279  */
281  instance->totem_interface->ip_port, &sockaddr, &addrlen);
282  memset(&msg_ucast, 0, sizeof(msg_ucast));
283  msg_ucast.msg_name = &sockaddr;
284  msg_ucast.msg_namelen = addrlen;
285  msg_ucast.msg_iov = (void *)&iovec;
286  msg_ucast.msg_iovlen = 1;
287 #ifdef HAVE_MSGHDR_CONTROL
288  msg_ucast.msg_control = 0;
289 #endif
290 #ifdef HAVE_MSGHDR_CONTROLLEN
291  msg_ucast.msg_controllen = 0;
292 #endif
293 #ifdef HAVE_MSGHDR_FLAGS
294  msg_ucast.msg_flags = 0;
295 #endif
296 #ifdef HAVE_MSGHDR_ACCRIGHTS
297  msg_ucast.msg_accrights = NULL;
298 #endif
299 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
300  msg_ucast.msg_accrightslen = 0;
301 #endif
302 
303 
304  /*
305  * Transmit unicast message
306  * An error here is recovered by totemsrp
307  */
308  res = sendmsg (instance->token_socket, &msg_ucast, MSG_NOSIGNAL);
309  if (res < 0) {
310  LOGSYS_PERROR (errno, instance->totemudpu_log_level_debug,
311  "sendmsg(ucast) failed (non-critical)");
312  }
313 }
314 
315 static inline void mcast_sendmsg (
316  struct totemudpu_instance *instance,
317  const void *msg,
318  unsigned int msg_len)
319 {
320  struct msghdr msg_mcast;
321  int res = 0;
322  size_t buf_out_len;
323  unsigned char buf_out[FRAME_SIZE_MAX];
324  struct iovec iovec;
325  struct sockaddr_storage sockaddr;
326  int addrlen;
327  struct list_head *list;
328  struct totemudpu_member *member;
329 
330  /*
331  * Encrypt and digest the message
332  */
334  instance->crypto_inst,
335  (const unsigned char *)msg,
336  msg_len,
337  buf_out,
338  &buf_out_len) != 0) {
339  log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
340  return;
341  }
342 
343  iovec.iov_base = (void *)buf_out;
344  iovec.iov_len = buf_out_len;
345 
346  memset(&msg_mcast, 0, sizeof(msg_mcast));
347  /*
348  * Build multicast message
349  */
350  for (list = instance->member_list.next;
351  list != &instance->member_list;
352  list = list->next) {
353 
354  member = list_entry (list,
355  struct totemudpu_member,
356  list);
357 
359  instance->totem_interface->ip_port, &sockaddr, &addrlen);
360  msg_mcast.msg_name = &sockaddr;
361  msg_mcast.msg_namelen = addrlen;
362  msg_mcast.msg_iov = (void *)&iovec;
363  msg_mcast.msg_iovlen = 1;
364  #ifdef HAVE_MSGHDR_CONTROL
365  msg_mcast.msg_control = 0;
366  #endif
367  #ifdef HAVE_MSGHDR_CONTROLLEN
368  msg_mcast.msg_controllen = 0;
369  #endif
370  #ifdef HAVE_MSGHDR_FLAGS
371  msg_mcast.msg_flags = 0;
372  #endif
373  #ifdef HAVE_MSGHDR_ACCRIGHTS
374  msg_mcast.msg_accrights = NULL;
375  #endif
376  #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
377  msg_mcast.msg_accrightslen = 0;
378  #endif
379 
380  /*
381  * Transmit multicast message
382  * An error here is recovered by totemsrp
383  */
384  res = sendmsg (member->fd, &msg_mcast, MSG_NOSIGNAL);
385  if (res < 0) {
386  LOGSYS_PERROR (errno, instance->totemudpu_log_level_debug,
387  "sendmsg(mcast) failed (non-critical)");
388  }
389  }
390 }
391 
393  void *udpu_context)
394 {
395  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
396  int res = 0;
397 
398  if (instance->token_socket > 0) {
399  qb_loop_poll_del (instance->totemudpu_poll_handle,
400  instance->token_socket);
401  close (instance->token_socket);
402  }
403 
404  return (res);
405 }
406 
407 static int net_deliver_fn (
408  int fd,
409  int revents,
410  void *data)
411 {
412  struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
413  struct msghdr msg_recv;
414  struct iovec *iovec;
415  struct sockaddr_storage system_from;
416  int bytes_received;
417  int res = 0;
418 
419  iovec = &instance->totemudpu_iov_recv;
420 
421  /*
422  * Receive datagram
423  */
424  msg_recv.msg_name = &system_from;
425  msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
426  msg_recv.msg_iov = iovec;
427  msg_recv.msg_iovlen = 1;
428 #ifdef HAVE_MSGHDR_CONTROL
429  msg_recv.msg_control = 0;
430 #endif
431 #ifdef HAVE_MSGHDR_CONTROLLEN
432  msg_recv.msg_controllen = 0;
433 #endif
434 #ifdef HAVE_MSGHDR_FLAGS
435  msg_recv.msg_flags = 0;
436 #endif
437 #ifdef HAVE_MSGHDR_ACCRIGHTS
438  msg_recv.msg_accrights = NULL;
439 #endif
440 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
441  msg_recv.msg_accrightslen = 0;
442 #endif
443 
444  bytes_received = recvmsg (fd, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
445  if (bytes_received == -1) {
446  return (0);
447  } else {
448  instance->stats_recv += bytes_received;
449  }
450 
451  /*
452  * Authenticate and if authenticated, decrypt datagram
453  */
454 
455  res = crypto_authenticate_and_decrypt (instance->crypto_inst, iovec->iov_base, &bytes_received);
456  if (res == -1) {
457  log_printf (instance->totemudpu_log_level_security, "Received message has invalid digest... ignoring.");
459  "Invalid packet data");
460  iovec->iov_len = FRAME_SIZE_MAX;
461  return 0;
462  }
463  iovec->iov_len = bytes_received;
464 
465  /*
466  * Handle incoming message
467  */
468  instance->totemudpu_deliver_fn (
469  instance->context,
470  iovec->iov_base,
471  iovec->iov_len);
472 
473  iovec->iov_len = FRAME_SIZE_MAX;
474  return (0);
475 }
476 
477 static int netif_determine (
478  struct totemudpu_instance *instance,
479  struct totem_ip_address *bindnet,
480  struct totem_ip_address *bound_to,
481  int *interface_up,
482  int *interface_num)
483 {
484  int res;
485 
486  res = totemip_iface_check (bindnet, bound_to,
487  interface_up, interface_num,
488  instance->totem_config->clear_node_high_bit);
489 
490 
491  return (res);
492 }
493 
494 
495 /*
496  * If the interface is up, the sockets for totem are built. If the interface is down
497  * this function is requeued in the timer list to retry building the sockets later.
498  */
499 static void timer_function_netif_check_timeout (
500  void *data)
501 {
502  struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
503  int interface_up;
504  int interface_num;
505  struct totem_ip_address *bind_address;
506 
507  /*
508  * Build sockets for every interface
509  */
510  netif_determine (instance,
511  &instance->totem_interface->bindnet,
512  &instance->totem_interface->boundto,
513  &interface_up, &interface_num);
514  /*
515  * If the network interface isn't back up and we are already
516  * in loopback mode, add timer to check again and return
517  */
518  if ((instance->netif_bind_state == BIND_STATE_LOOPBACK &&
519  interface_up == 0) ||
520 
521  (instance->my_memb_entries == 1 &&
522  instance->netif_bind_state == BIND_STATE_REGULAR &&
523  interface_up == 1)) {
524 
525  qb_loop_timer_add (instance->totemudpu_poll_handle,
526  QB_LOOP_MED,
527  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
528  (void *)instance,
529  timer_function_netif_check_timeout,
530  &instance->timer_netif_check_timeout);
531 
532  /*
533  * Add a timer to check for a downed regular interface
534  */
535  return;
536  }
537 
538  if (instance->token_socket > 0) {
539  qb_loop_poll_del (instance->totemudpu_poll_handle,
540  instance->token_socket);
541  close (instance->token_socket);
542  }
543 
544  if (interface_up == 0) {
545  /*
546  * Interface is not up
547  */
549  bind_address = &localhost;
550 
551  /*
552  * Add a timer to retry building interfaces and request memb_gather_enter
553  */
554  qb_loop_timer_add (instance->totemudpu_poll_handle,
555  QB_LOOP_MED,
556  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
557  (void *)instance,
558  timer_function_netif_check_timeout,
559  &instance->timer_netif_check_timeout);
560  } else {
561  /*
562  * Interface is up
563  */
565  bind_address = &instance->totem_interface->bindnet;
566  }
567  /*
568  * Create and bind the multicast and unicast sockets
569  */
570  totemudpu_build_sockets (instance,
571  bind_address,
572  &instance->totem_interface->boundto);
573 
574  qb_loop_poll_add (instance->totemudpu_poll_handle,
575  QB_LOOP_MED,
576  instance->token_socket,
577  POLLIN, instance, net_deliver_fn);
578 
579  totemip_copy (&instance->my_id, &instance->totem_interface->boundto);
580 
581  /*
582  * This reports changes in the interface to the user and totemsrp
583  */
584  if (instance->netif_bind_state == BIND_STATE_REGULAR) {
585  if (instance->netif_state_report & NETIF_STATE_REPORT_UP) {
587  "The network interface [%s] is now up.",
588  totemip_print (&instance->totem_interface->boundto));
590  instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
591  }
592  /*
593  * Add a timer to check for interface going down in single membership
594  */
595  if (instance->my_memb_entries == 1) {
596  qb_loop_timer_add (instance->totemudpu_poll_handle,
597  QB_LOOP_MED,
598  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
599  (void *)instance,
600  timer_function_netif_check_timeout,
601  &instance->timer_netif_check_timeout);
602  }
603 
604  } else {
607  "The network interface is down.");
608  instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
609  }
611 
612  }
613 }
614 
615 /* Set the socket priority to INTERACTIVE to ensure
616  that our messages don't get queued behind anything else */
617 static void totemudpu_traffic_control_set(struct totemudpu_instance *instance, int sock)
618 {
619 #ifdef SO_PRIORITY
620  int prio = 6; /* TC_PRIO_INTERACTIVE */
621 
622  if (setsockopt(sock, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(int))) {
623  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
624  "Could not set traffic priority");
625  }
626 #endif
627 }
628 
629 static int totemudpu_build_sockets_ip (
630  struct totemudpu_instance *instance,
631  struct totem_ip_address *bindnet_address,
632  struct totem_ip_address *bound_to,
633  int interface_num)
634 {
635  struct sockaddr_storage sockaddr;
636  int addrlen;
637  int res;
638  unsigned int recvbuf_size;
639  unsigned int optlen = sizeof (recvbuf_size);
640 
641  /*
642  * Setup unicast socket
643  */
644  instance->token_socket = socket (bindnet_address->family, SOCK_DGRAM, 0);
645  if (instance->token_socket == -1) {
646  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
647  "socket() failed");
648  return (-1);
649  }
650 
651  totemip_nosigpipe (instance->token_socket);
652  res = fcntl (instance->token_socket, F_SETFL, O_NONBLOCK);
653  if (res == -1) {
654  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
655  "Could not set non-blocking operation on token socket");
656  return (-1);
657  }
658 
659  /*
660  * Bind to unicast socket used for token send/receives
661  * This has the side effect of binding to the correct interface
662  */
663  totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port, &sockaddr, &addrlen);
664  res = bind (instance->token_socket, (struct sockaddr *)&sockaddr, addrlen);
665  if (res == -1) {
666  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
667  "bind token socket failed");
668  return (-1);
669  }
670 
671  /*
672  * the token_socket can receive many messages. Allow a large number
673  * of receive messages on this socket
674  */
675  recvbuf_size = MCAST_SOCKET_BUFFER_SIZE;
676  res = setsockopt (instance->token_socket, SOL_SOCKET, SO_RCVBUF,
677  &recvbuf_size, optlen);
678  if (res == -1) {
679  LOGSYS_PERROR (errno, instance->totemudpu_log_level_notice,
680  "Could not set recvbuf size");
681  }
682 
683  return 0;
684 }
685 
686 static int totemudpu_build_sockets (
687  struct totemudpu_instance *instance,
688  struct totem_ip_address *bindnet_address,
689  struct totem_ip_address *bound_to)
690 {
691  int interface_num;
692  int interface_up;
693  int res;
694 
695  /*
696  * Determine the ip address bound to and the interface name
697  */
698  res = netif_determine (instance,
699  bindnet_address,
700  bound_to,
701  &interface_up,
702  &interface_num);
703 
704  if (res == -1) {
705  return (-1);
706  }
707 
708  totemip_copy(&instance->my_id, bound_to);
709 
710  res = totemudpu_build_sockets_ip (instance,
711  bindnet_address, bound_to, interface_num);
712 
713  /* We only send out of the token socket */
714  totemudpu_traffic_control_set(instance, instance->token_socket);
715 
716  /*
717  * Rebind all members to new ips
718  */
720 
721  return res;
722 }
723 
724 /*
725  * Totem Network interface - also does encryption/decryption
726  * depends on poll abstraction, POSIX, IPV4
727  */
728 
729 /*
730  * Create an instance
731  */
733  qb_loop_t *poll_handle,
734  void **udpu_context,
735  struct totem_config *totem_config,
736  totemsrp_stats_t *stats,
737  int interface_no,
738  void *context,
739 
740  void (*deliver_fn) (
741  void *context,
742  const void *msg,
743  unsigned int msg_len),
744 
745  void (*iface_change_fn) (
746  void *context,
747  const struct totem_ip_address *iface_address),
748 
749  void (*target_set_completed) (
750  void *context))
751 {
752  struct totemudpu_instance *instance;
753 
754  instance = malloc (sizeof (struct totemudpu_instance));
755  if (instance == NULL) {
756  return (-1);
757  }
758 
759  totemudpu_instance_initialize (instance);
760 
761  instance->totem_config = totem_config;
762  instance->stats = stats;
763 
764  /*
765  * Configure logging
766  */
767  instance->totemudpu_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
774 
775  /*
776  * Initialize random number generator for later use to generate salt
777  */
778  instance->crypto_inst = crypto_init (totem_config->private_key,
779  totem_config->private_key_len,
780  totem_config->crypto_cipher_type,
781  totem_config->crypto_hash_type,
782  instance->totemudpu_log_printf,
784  instance->totemudpu_log_level_notice,
785  instance->totemudpu_log_level_error,
786  instance->totemudpu_subsys_id);
787  if (instance->crypto_inst == NULL) {
788  free(instance);
789  return (-1);
790  }
791  /*
792  * Initialize local variables for totemudpu
793  */
794  instance->totem_interface = &totem_config->interfaces[interface_no];
795  memset (instance->iov_buffer, 0, FRAME_SIZE_MAX);
796 
797  instance->totemudpu_poll_handle = poll_handle;
798 
799  instance->totem_interface->bindnet.nodeid = instance->totem_config->node_id;
800 
801  instance->context = context;
802  instance->totemudpu_deliver_fn = deliver_fn;
803 
804  instance->totemudpu_iface_change_fn = iface_change_fn;
805 
806  instance->totemudpu_target_set_completed = target_set_completed;
807 
808  totemip_localhost (AF_INET, &localhost);
809  localhost.nodeid = instance->totem_config->node_id;
810 
811  /*
812  * RRP layer isn't ready to receive message because it hasn't
813  * initialized yet. Add short timer to check the interfaces.
814  */
815  qb_loop_timer_add (instance->totemudpu_poll_handle,
816  QB_LOOP_MED,
817  100*QB_TIME_NS_IN_MSEC,
818  (void *)instance,
819  timer_function_netif_check_timeout,
820  &instance->timer_netif_check_timeout);
821 
822  *udpu_context = instance;
823  return (0);
824 }
825 
827 {
828  return malloc (FRAME_SIZE_MAX);
829 }
830 
831 void totemudpu_buffer_release (void *ptr)
832 {
833  return free (ptr);
834 }
835 
837  void *udpu_context,
838  int processor_count)
839 {
840  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
841  int res = 0;
842 
843  instance->my_memb_entries = processor_count;
844  qb_loop_timer_del (instance->totemudpu_poll_handle,
845  instance->timer_netif_check_timeout);
846  if (processor_count == 1) {
847  qb_loop_timer_add (instance->totemudpu_poll_handle,
848  QB_LOOP_MED,
849  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
850  (void *)instance,
851  timer_function_netif_check_timeout,
852  &instance->timer_netif_check_timeout);
853  }
854 
855  return (res);
856 }
857 
858 int totemudpu_recv_flush (void *udpu_context)
859 {
860  int res = 0;
861 
862  return (res);
863 }
864 
865 int totemudpu_send_flush (void *udpu_context)
866 {
867  int res = 0;
868 
869  return (res);
870 }
871 
873  void *udpu_context,
874  const void *msg,
875  unsigned int msg_len)
876 {
877  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
878  int res = 0;
879 
880  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
881 
882  return (res);
883 }
885  void *udpu_context,
886  const void *msg,
887  unsigned int msg_len)
888 {
889  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
890  int res = 0;
891 
892  mcast_sendmsg (instance, msg, msg_len);
893 
894  return (res);
895 }
896 
898  void *udpu_context,
899  const void *msg,
900  unsigned int msg_len)
901 {
902  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
903  int res = 0;
904 
905  mcast_sendmsg (instance, msg, msg_len);
906 
907  return (res);
908 }
909 
910 extern int totemudpu_iface_check (void *udpu_context)
911 {
912  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
913  int res = 0;
914 
915  timer_function_netif_check_timeout (instance);
916 
917  return (res);
918 }
919 
920 extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *totem_config)
921 {
922 #define UDPIP_HEADER_SIZE (20 + 8) /* 20 bytes for ip 8 bytes for udp */
923  totem_config->net_mtu -= crypto_sec_header_size(totem_config->crypto_cipher_type,
924  totem_config->crypto_hash_type) +
926 }
927 
928 const char *totemudpu_iface_print (void *udpu_context) {
929  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
930  const char *ret_char;
931 
932  ret_char = totemip_print (&instance->my_id);
933 
934  return (ret_char);
935 }
936 
938  void *udpu_context,
939  struct totem_ip_address *addr)
940 {
941  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
942  int res = 0;
943 
944  memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
945 
946  return (res);
947 }
948 
950  void *udpu_context,
951  const struct totem_ip_address *token_target)
952 {
953  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
954  int res = 0;
955 
956  memcpy (&instance->token_target, token_target,
957  sizeof (struct totem_ip_address));
958 
959  instance->totemudpu_target_set_completed (instance->context);
960 
961  return (res);
962 }
963 
965  void *udpu_context)
966 {
967  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
968  unsigned int res;
969  struct sockaddr_storage system_from;
970  struct msghdr msg_recv;
971  struct pollfd ufd;
972  int nfds;
973  int msg_processed = 0;
974 
975  /*
976  * Receive datagram
977  */
978  msg_recv.msg_name = &system_from;
979  msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
980  msg_recv.msg_iov = &instance->totemudpu_iov_recv;
981  msg_recv.msg_iovlen = 1;
982 #ifdef HAVE_MSGHDR_CONTROL
983  msg_recv.msg_control = 0;
984 #endif
985 #ifdef HAVE_MSGHDR_CONTROLLEN
986  msg_recv.msg_controllen = 0;
987 #endif
988 #ifdef HAVE_MSGHDR_FLAGS
989  msg_recv.msg_flags = 0;
990 #endif
991 #ifdef HAVE_MSGHDR_ACCRIGHTS
992  msg_recv.msg_accrights = NULL;
993 #endif
994 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
995  msg_recv.msg_accrightslen = 0;
996 #endif
997 
998  do {
999  ufd.fd = instance->token_socket;
1000  ufd.events = POLLIN;
1001  nfds = poll (&ufd, 1, 0);
1002  if (nfds == 1 && ufd.revents & POLLIN) {
1003  res = recvmsg (instance->token_socket, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
1004  if (res != -1) {
1005  msg_processed = 1;
1006  } else {
1007  msg_processed = -1;
1008  }
1009  }
1010  } while (nfds == 1);
1011 
1012  return (msg_processed);
1013 }
1014 
1015 static int totemudpu_create_sending_socket(
1016  void *udpu_context,
1017  const struct totem_ip_address *member)
1018 {
1019  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1020  int fd;
1021  int res;
1022  unsigned int sendbuf_size;
1023  unsigned int optlen = sizeof (sendbuf_size);
1024  struct sockaddr_storage sockaddr;
1025  int addrlen;
1026 
1027  fd = socket (member->family, SOCK_DGRAM, 0);
1028  if (fd == -1) {
1029  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1030  "Could not create socket for new member");
1031  return (-1);
1032  }
1033  totemip_nosigpipe (fd);
1034  res = fcntl (fd, F_SETFL, O_NONBLOCK);
1035  if (res == -1) {
1036  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1037  "Could not set non-blocking operation on token socket");
1038  goto error_close_fd;
1039  }
1040 
1041  /*
1042  * These sockets are used to send multicast messages, so their buffers
1043  * should be large
1044  */
1045  sendbuf_size = MCAST_SOCKET_BUFFER_SIZE;
1046  res = setsockopt (fd, SOL_SOCKET, SO_SNDBUF,
1047  &sendbuf_size, optlen);
1048  if (res == -1) {
1049  LOGSYS_PERROR (errno, instance->totemudpu_log_level_notice,
1050  "Could not set sendbuf size");
1051  /*
1052  * Fail in setting sendbuf size is not fatal -> don't exit
1053  */
1054  }
1055 
1056  /*
1057  * Bind to sending interface
1058  */
1059  totemip_totemip_to_sockaddr_convert(&instance->my_id, 0, &sockaddr, &addrlen);
1060  res = bind (fd, (struct sockaddr *)&sockaddr, addrlen);
1061  if (res == -1) {
1062  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1063  "bind token socket failed");
1064  goto error_close_fd;
1065  }
1066 
1067  return (fd);
1068 
1069 error_close_fd:
1070  close(fd);
1071  return (-1);
1072 }
1073 
1075  void *udpu_context,
1076  const struct totem_ip_address *member)
1077 {
1078  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1079 
1080  struct totemudpu_member *new_member;
1081 
1082  new_member = malloc (sizeof (struct totemudpu_member));
1083  if (new_member == NULL) {
1084  return (-1);
1085  }
1086  log_printf (LOGSYS_LEVEL_NOTICE, "adding new UDPU member {%s}",
1087  totemip_print(member));
1088  list_init (&new_member->list);
1089  list_add_tail (&new_member->list, &instance->member_list);
1090  memcpy (&new_member->member, member, sizeof (struct totem_ip_address));
1091  new_member->fd = totemudpu_create_sending_socket(udpu_context, member);
1092 
1093  return (0);
1094 }
1095 
1097  void *udpu_context,
1098  const struct totem_ip_address *token_target)
1099 {
1100  int found = 0;
1101  struct list_head *list;
1102  struct totemudpu_member *member;
1103 
1104  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1105 
1106  /*
1107  * Find the member to remove and close its socket
1108  */
1109  for (list = instance->member_list.next;
1110  list != &instance->member_list;
1111  list = list->next) {
1112 
1113  member = list_entry (list,
1114  struct totemudpu_member,
1115  list);
1116 
1117  if (totemip_compare (token_target, &member->member)==0) {
1119  "removing UDPU member {%s}",
1120  totemip_print(&member->member));
1121 
1122  if (member->fd > 0) {
1124  "Closing socket to: {%s}",
1125  totemip_print(&member->member));
1126  qb_loop_poll_del (instance->totemudpu_poll_handle,
1127  member->fd);
1128  close (member->fd);
1129  }
1130  found = 1;
1131  break;
1132  }
1133  }
1134 
1135  /*
1136  * Delete the member from the list
1137  */
1138  if (found) {
1139  list_del (list);
1140  }
1141 
1142  instance = NULL;
1143  return (0);
1144 }
1145 
1147  void *udpu_context)
1148 {
1149  struct list_head *list;
1150  struct totemudpu_member *member;
1151 
1152  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1153 
1154  for (list = instance->member_list.next;
1155  list != &instance->member_list;
1156  list = list->next) {
1157 
1158  member = list_entry (list,
1159  struct totemudpu_member,
1160  list);
1161 
1162  if (member->fd > 0) {
1163  close (member->fd);
1164  }
1165 
1166  member->fd = totemudpu_create_sending_socket(udpu_context, &member->member);
1167  }
1168 
1169  return (0);
1170 }