modules/ud/ud_process_stream.c

/* [<][>]
[^][v][top][bottom][index][help] */

FUNCTIONS

This source file includes following functions.
  1. get_NRTM_stream
  2. report_transaction
  3. s_split
  4. escape_apostrophes
  5. line_type
  6. process_nrtm
  7. process_updates
  8. UD_process_stream

/***************************************
  $Revision: 1.7 $

  Functions to process data stream( file, network socket, etc.)

  Status: NOT REVUED, NOT TESTED

 Author(s):       Chris Ottrey, Andrei Robachevsky

  ******************/ /******************
  Modification History:
        andrei (17/01/2000) Created.
  ******************/ /******************
  Copyright (c) 2000                              RIPE NCC
 
  All Rights Reserved
  
  Permission to use, copy, modify, and distribute this software and its
  documentation for any purpose and without fee is hereby granted,
  provided that the above copyright notice appear in all copies and that
  both that copyright notice and this permission notice appear in
  supporting documentation, and that the name of the author not be
  used in advertising or publicity pertaining to distribution of the
  software without specific, written prior permission.
  
  THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
  ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
  AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 ***************************************/
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "ud.h"
#include "ud_int.h"

typedef enum _Line_Type_t {
 LINE_ATTRIBUTE,
 LINE_COMMENT,
 LINE_EMPTY,
 LINE_EOF,
 LINE_ADD,
 LINE_UPD,
 LINE_DEL
} Line_Type_t;

/* Maximum number of objects(serials) we can consume at a time */
#define SBUNCH  1000

static int report_transaction(Transaction_t *tr, Log_t *log, char *obj_name, char *reason);
static char *s_split(char *line);

static int process_nrtm(Transaction_t *tr, struct _nrtm *nrtm, Log_t *log, char *object_name, int operation);
static int process_updates(Transaction_t *tr, Log_t *log, char *object_name, int operation, int standalone);




/* temporary files to download serials */              
char tmpfile1[STR_S], tmpfile2[STR_S];

FILE *get_NRTM_stream(struct _nrtm *nrtm, int upto_last)
/* [<][>][^][v][top][bottom][index][help] */
{
int sockfd;
struct hostent *hptr;
struct sockaddr_in serv_addr;
struct in_addr  *paddr;
char line_buff[STR_XXL];
FILE *fp;
int fd;
int fdtmp;
int nread, nwrite;
struct hostent result;
int error;


 fprintf(stderr, "Making connection to NRTM server ...\n");
 if ((sockfd=socket(AF_INET, SOCK_STREAM, 0))==-1){
   perror("socket");
   return(NULL);
 }  
// hptr=gethostbyname(nrtm->server);
 hptr=gethostbyname_r(nrtm->server,  &result, line_buff, sizeof(line_buff), &error);
 if (hptr) { // this is a network stream
   paddr=(struct in_addr *)hptr->h_addr;
   bzero(&serv_addr, sizeof(serv_addr));
   serv_addr.sin_family=AF_INET;
   serv_addr.sin_port=nrtm->port;
   memcpy(&serv_addr.sin_addr, paddr, sizeof(struct in_addr));
   fprintf(stderr,"Trying %s port %d\n", inet_ntoa(serv_addr.sin_addr), nrtm->port);
   if(connect(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr))==-1) { 
     perror("connect");
     return(NULL);
   }  
   fprintf(stderr, "Sending Invitation\n");
   if(upto_last)
      sprintf(line_buff, "-g RIPE:%d:%d-LAST\n", nrtm->version, nrtm->current_serial+1);
   else
      sprintf(line_buff, "-g RIPE:%d:%d-%d\n", nrtm->version, nrtm->current_serial+1, nrtm->current_serial+SBUNCH);   
   write(sockfd, line_buff, strlen(line_buff));
   fd=sockfd;
   fprintf(stderr, "%s", line_buff);
   fprintf(stderr, "Returning stream pointer\n");
 }
 else { // this is a file stream
   fprintf(stderr, "Trying file ...\n");
   if((fd=open(nrtm->server, O_RDONLY, 0666))==-1) {
     perror("open");
     return(NULL);
   }  
 }  

/* make temporary files */
 sprintf(tmpfile1, "temp.XXXX");

 if((fdtmp=mkstemp(tmpfile1))==-1) {
   perror("mkstemp");
   close(fd);
   return(NULL);
 }  
 while ((nread=read(fd, line_buff, sizeof(line_buff)))) {
   if(nread==-1) return(NULL);
   nwrite=write(fdtmp, line_buff,nread);
   if(nread != nwrite) return(NULL);
}
 
 close(fd); close(fdtmp);
 
 sprintf(tmpfile2, "%s.2", tmpfile1);
 
 sprintf(line_buff, "cat %s | ./ripe2rpsl > %s", tmpfile1, tmpfile2);
 if (system(line_buff)!=0) return(NULL);
 if((fp=fopen(tmpfile2, "r+"))==NULL)return(NULL);
 
 unlink(tmpfile1);
 
 return(fp);
 
}

/******************************************************************
* report_transaction()                                            *
*                                                                 * 
* Prints error report to the log                                  *
*                                                                 *
* reason - additional message thet will be included               *
*                                                                 *
* *****************************************************************/
static int report_transaction(Transaction_t *tr, Log_t *log, char *obj_name, char *reason)
/* [<][>][^][v][top][bottom][index][help] */
{
int result=0;

 if(tr->succeeded==0) {
  result=tr->error;
  log->num_failed++;
  printf("FAILED[%s][%s(%d)](%d/%d)\n ", obj_name, reason, result, log->num_failed, (log->num_failed)+(log->num_ok));   
  fprintf(log->logfile, "*FAILED[%s][%s](%d/%d)\n ", obj_name, reason, log->num_failed, (log->num_failed)+(log->num_ok));
  if(result & ERROR_U_MEM) fprintf(log->logfile, "\t*Memory allocation error\n");
  if(result & ERROR_U_DBS) fprintf(log->logfile, "\t*Database (SQL) error\n");
  if(result & ERROR_U_OBJ) fprintf(log->logfile, "\t*Object (RF) error\n");
  if(result & ERROR_U_AUT) fprintf(log->logfile, "\t*Object authentication error\n");
  if(result & ERROR_U_BADOP) fprintf(log->logfile, "\t*Bad operation\n");
  if(result & ERROR_U_COP) fprintf(log->logfile, "\t*Conflicting operation\n");
  if(result & ERROR_U_NSUP) fprintf(log->logfile, "\t*Object of this type is not supported\n");
  if(result & ERROR_U_BUG) fprintf(log->logfile, "\t*Software bug - report to <ripe-dbm@ripe.net>\n");
  fprintf(log->logfile, "%s", (tr->error_script)->str);
  result=(-1)*result;                                                
  fflush(log->logfile);
 }
 else {
  result=1;
  log->num_ok++;
  printf("OK(%d/%d)\n", log->num_ok, (log->num_failed)+(log->num_ok));
 }
                                                                                                                                                
 return(result);
}

/******************************************************************
* char *s_split(char *line)                                       *
* consequently returns words (separated by whitespace in the line)*
* NULL - end. You need to retreive all words !                    *
*                                                                 *
* *****************************************************************/
static char *s_split(char *line)
/* [<][>][^][v][top][bottom][index][help] */
{
static char *delim;
static char *token=NULL;

 if(token==NULL)token=line;
 else token=delim;
 
 if(token==NULL)return(token);
 while(isspace((int)*token))token++;
 delim=token;
 
 while(!isspace((int)*delim)) {
        if((*delim)=='\0'){
                if(delim==token)token=NULL;
                delim=NULL; return(token);
        }
        delim++;
 }
 *delim='\0'; delim++;
 return(token);

}



GString *escape_apostrophes(GString *text) {
/* [<][>][^][v][top][bottom][index][help] */
  int i;
  for (i=0; i < text->len; i++) {
    if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
      text = g_string_insert_c(text, i, '\\');
      i++;
    }
  }
 return(text); 
} /* escape_apostrophes() */

static Line_Type_t line_type(const char *line) {
/* [<][>][^][v][top][bottom][index][help] */
  Line_Type_t result = -1;

  if (strncmp(line, "# EOF", 4) == 0) {
    result = LINE_EOF;
  }
  else if (strncmp(line, "#", 1) == 0) {
    result = LINE_COMMENT;
  }
  else if (strcmp(line, "\n") == 0) {
    result = LINE_EMPTY;
  }
  else if (strncmp(line, "ADD", 3) == 0) {
    result = LINE_ADD;
  }
  else if (strncmp(line, "UPD", 3) == 0) {
    result = LINE_UPD;
  }
  else if (strncmp(line, "DEL", 3) == 0) {
    result = LINE_DEL;
  }
  else {
    result = LINE_ATTRIBUTE;
  }

  return result;
} /* line_type() */




/************************************************************
* process_nrtm()                                            *
*                                                           *
* Process object in NRTM client mode                        *
*                                                           *
* nrtm - pointer to _nrtm structure                         *
* log - pointer to Log_t structure                          *
* object_name - name of the object                          * 
* operation - operation code (OP_ADD/OP_DEL)                *
*                                                           *
* Returns:                                                  *
* 1  - okay                                                 *
* <0 - error                                                *
*                                                           *
************************************************************/

static int process_nrtm(Transaction_t *tr, struct _nrtm *nrtm, Log_t *log, char *object_name, int operation)
/* [<][>][^][v][top][bottom][index][help] */
{
int error=0;
//fprintf(stderr,"NRTM mode\n");
  
  switch (operation) {
  
  case OP_ADD:
    if(nrtm->tr){ //saved?
      if(tr->object_id==0) {
//      fprintf(stderr,"DEL previous\n");
        object_process(nrtm->tr); // delete the previous(saved) object
        error=report_transaction(nrtm->tr, log, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");
        object_free(nrtm->tr->object);
        transaction_free(nrtm->tr); nrtm->tr=NULL;
        tr->action=TR_CREATE;
//      fprintf(stderr,"CREATE next\n"); 
        object_process(tr);     // create a new one
        error=report_transaction(tr, log, object_name, "NRTM:ADD:While creating new");
      }
      else { //compare the two, may be we may collapse operations
        if(tr->object_id==nrtm->tr->object_id) {
          object_free(nrtm->tr->object);
          transaction_free(nrtm->tr); nrtm->tr=NULL;
//        fprintf(stderr,"DEL-ADD ->> UPDATE\n");
          tr->action=TR_UPDATE;
          object_process(tr);
          report_transaction(tr, log, object_name,"NRTM:upd");
          error=report_transaction(tr, log, object_name,"NRTM:upd");
        }
        else { // this should be a dummy object in the database or an interleaved operation
//        fprintf(stderr,"DEL previous\n");
          object_process(nrtm->tr); // delete the previous(saved) object
          error=report_transaction(nrtm->tr, log, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");
          object_free(nrtm->tr->object);
          transaction_free(nrtm->tr); nrtm->tr=NULL;
          tr->action=TR_UPDATE;
//        fprintf(stderr,"UPDATE next(dummy)\n"); 
          object_process(tr);   // create a new one
          error=report_transaction(tr, log, object_name, "NRTM:ADD:While creating new");
        }
      }
    }
    else { // brand new object
      if(tr->object_id==0) {
//      fprintf(stderr,"CREATE new\n");
        tr->action=TR_CREATE;
        object_process(tr);
        error=report_transaction(tr, log, object_name,"NRTM:ADD:While creating new");
      }
      else { // this may happen because of dummies
//      fprintf(stderr,"CREATE new\n");
        tr->action=TR_UPDATE;
        object_process(tr);
        error=report_transaction(tr, log, object_name,"NRTM:ADD:While creating new");
      } 
    }
    break;
    
  case OP_DEL:
    if(nrtm->tr){ //saved?
//    fprintf(stderr,"DEL previous\n");
      object_process(nrtm->tr); // delete the previous(saved) object
      error=report_transaction(nrtm->tr, log, nrtm->object_name, "NRTM:DEL:While deleting previous(saved) object");
      object_free(nrtm->tr->object);
      transaction_free(nrtm->tr); nrtm->tr=NULL;
    }
    if(tr->object_id>0){ // save the object
//    fprintf(stderr,"SAVE del object\n");
      tr->action=TR_DELETE;
      nrtm->tr=tr;
      strcpy(nrtm->object_name, object_name);
      return(error);
    }
    else { // this is an error
      tr->succeeded=0; tr->error|=ERROR_U_COP;
      error=report_transaction(tr, log, object_name, "NRTM:OOS:Trying to DEL non-existing object");
//    fprintf(stderr,"Lost sync. Skipping\n");
    }
    break;
  
  default:
    tr->succeeded=0; tr->error |=ERROR_U_BADOP;
    break;  
  }
  
  object_free(tr->object);
  transaction_free(tr);
  return(error);
} /* process_nrtm() */



/************************************************************
* process_updates()                                         *
*                                                           *
* Process object in update mode                             *
*                                                           *
* nrtm - pointer to _nrtm structure                         *
* log - pointer to Log_t structure                          *
* object_name - name of the object                          *
* operation - operation code (OP_ADD/OP_DEL)                *
* standalone - if the program is executed standalone        *
*                                                           *
* Returns:                                                  *
* 1  - okay                                                 *
* <0 - error                                                *
*                                                           * 
************************************************************/

static int process_updates(Transaction_t *tr, Log_t *log, char *object_name, int operation, int standalone)
/* [<][>][^][v][top][bottom][index][help] */
{
int error=0;

    switch(operation) {
    
    case OP_ADD:
    case OP_UPD:
             if(tr->object_id==0) tr->action=TR_CREATE; else tr->action=TR_UPDATE;
             object_process(tr);
             break;

    case OP_DEL:               
             if(tr->object_id==0) { // trying t delete non-existing object
               tr->succeeded=0; tr->error|=ERROR_U_COP;
             } else {
               tr->action=TR_DELETE;
               object_process(tr);
             }
             break;
                       
    default:                   
      /* bad operation for this mode if not standalone */
             if(standalone) {
               if(tr->object_id==0)tr->action=TR_CREATE; else tr->action=TR_UPDATE;
               object_process(tr);
             }
             else {
               tr->succeeded=0; 
               tr->error|=ERROR_U_BADOP; 
             }
             break;
    }
    error=report_transaction(tr, log, object_name, "UPD");
    
    return(error);
               
} /* process_updates() */

/************************************************************
*                                                           *
* int UD_process_stream(UD_stream_t *ud_stream)             *
*                                                           *
* Processes the stream                                      *
*                                                           *
* ud_stream - pointer to UD_stream_t structure              *
*                                                           *
* Returns:                                                  *
* in update mode (!standalone)(1 object processed):         *
* 1 - no error                                              *
* <0- errors                                                *
*                                                           *
* in NRTM & standalone modes                                *
* total number of object processed                          *
*                                                           *
************************************************************/

int UD_process_stream(UD_stream_t *ud_stream)
/* [<][>][^][v][top][bottom][index][help] */
{
  FILE *logfile;
  char line_buff[STR_XXL], object_name[STR_XXL];
  GString *g_line_buff; // needed to escape apostrophes

  Attribute_t *attr, *attr_split;
  Attribute_t *mnt_by; // we need this for reordering mnt_by and member_of (member_of should come after)
  Attribute_t *nic_hdl; // we need this for reordering nic_hdl and admin_c, etc. (admin_c should come after)
  Object_t *obj = NULL;
  Transaction_t *tr = NULL;
  SQ_connection_t *sql_connection;
  int start_object;
  int a_type;
  char *s_attr;
  long num_skip;
  struct _nrtm *nrtm;
  Log_t log;
  time_t stime, ftime;
  double obj_second1, obj_second10;
  int standalone;
  int error;
  int operation=0;
  
  nrtm=ud_stream->nrtm;
  standalone=IS_STANDALONE(ud_stream->ud_mode);
  start_object = 1;
  
 if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){ fprintf(stderr, "E: cannot allocate gstring\n"); return(-1); }



fprintf(stderr, "D: Making SQL connection to %s@%s ...", ud_stream->db_name, ud_stream->db_host);
  sql_connection = SQ_get_connection(ud_stream->db_host, ud_stream->db_port, 
                                        ud_stream->db_name, ud_stream->db_user, ud_stream->db_pswd);
  if(!sql_connection) {
   fprintf(stderr, "D: ERROR: no SQL connection\n");
   return(-1);
  }     
fprintf(stderr, "OK\n");

  logfile = fopen(ud_stream->log, "a+");
  log.logfile=logfile;
  log.num_ok=0; log.num_failed=0;

/* This is useful for loading DB from huge disk file. */
  num_skip=ud_stream->num_skip;
  if(num_skip>0) fprintf(stderr, "skipping %lu records\n", num_skip);

 stime=time(NULL);

        
  while (fgets(line_buff, STR_XXL, ud_stream->stream) != NULL) {

    switch (line_type(line_buff)) {
      case LINE_ATTRIBUTE:
        if (start_object == 1) {
          if(num_skip>0){ printf("\r%10lu", num_skip); num_skip--; log.num_ok++; break; } 
                
          mnt_by=NULL;
          nic_hdl=NULL;
          strncpy(object_name, line_buff, strlen(line_buff)-1); 
          *(object_name+strlen(line_buff)-1)='\0';
          obj = object_new(line_buff);
          if (obj) {
            start_object = 0;
            printf("D: object: [%s] ", object_name);
          } 
        }
        if (obj != NULL) {
          g_string_sprintf(g_line_buff, "%s", line_buff);
          g_line_buff=escape_apostrophes(g_line_buff);
          attr = attribute_new(g_line_buff->str);
          g_string_sprintfa(obj->object, "%s", g_line_buff->str);
          if (attr != NULL) {
           switch (a_type=(attr->type)) {
            case A_MB:  mnt_by=attr;
                        break;
            case A_NH:  nic_hdl=attr;
                        break;
            case A_PN:
            case A_RO:
            case A_MR:
            case A_SD:
            case A_RZ:
            case A_NS: //these attributes may appear several on the line - split them
                while((s_attr=s_split(attr->value))){
                  attr_split = attribute_new1(a_type, s_attr);
                  obj->attributes = g_slist_append(obj->attributes, attr_split);
                }
                attribute_free(attr, NULL);
                attr=NULL;
                break;  
            default:    break;          
           }
            if(attr){ obj->attributes = g_slist_append(obj->attributes, attr);  
            }
          }
        } 
      break;

      case LINE_COMMENT:
      break;

      case LINE_EOF:
      break;
      
      case LINE_ADD:
        operation=OP_ADD;
      break;
      
      case LINE_UPD:
        operation=OP_UPD;
      break;  
      
      case LINE_DEL:
        operation=OP_DEL;
      break;    

      case LINE_EMPTY:
        start_object=1;
        if (obj != NULL) {
        /* reorder some attributes */
          if(mnt_by){
                obj->attributes = g_slist_remove(obj->attributes, mnt_by);
                obj->attributes = g_slist_insert(obj->attributes, mnt_by, 1);
          }
          if(nic_hdl){
                obj->attributes = g_slist_remove(obj->attributes, nic_hdl);
                obj->attributes = g_slist_insert(obj->attributes, nic_hdl, 1);
          }
          tr = transaction_new(sql_connection, obj->type);
          if (tr != NULL) {
            tr->standalone=standalone;
            tr->dummy=IS_DUMMY_ALLOWED(ud_stream->ud_mode);
            tr->load_pass=ud_stream->load_pass;
            tr->object=obj;
            if(ud_stream->load_pass) { tr->thread_ins=0; tr->thread_upd=0; }
            if(ud_stream->load_pass==1) tr->object_id=0;
            else tr->object_id=get_object_id(tr);
            if(tr->object_id==-1) { // some error
              tr->succeeded=0;
              report_transaction(tr, &log, object_name, "DB error");
              transaction_free(tr); tr=NULL;
              object_free(obj); obj=NULL;
              operation=OP_NOOP;
              break;
            }
            else 
             if(nrtm) {         //NRTM mode
               error=process_nrtm(tr, nrtm, &log, object_name, operation);
               tr=NULL;
               obj=NULL;
               operation=OP_NOOP;
             }
             else {     //update mode
               error=process_updates(tr, &log, object_name, operation, standalone);
               if(!standalone) { /* copy the script */
                ud_stream->error_script=g_strdup((tr->error_script)->str);
                object_free(tr->object); obj=NULL;
                transaction_free(tr); tr=NULL;
                g_string_free(g_line_buff, TRUE);
                SQ_close_connection(sql_connection);
                fclose(logfile);
                return(error);
               }
               else {
                object_free(tr->object); obj=NULL;
                transaction_free(tr); tr=NULL;
                operation=OP_NOOP;
               } 
             }
            /* this is a good place for quick inerrupt */
          }  
        }
      break;

      default:
        fprintf(stderr, "ERROR: Bad line type\n");
    } /* switch */
  } /* while */
 
  if(nrtm)
   if(nrtm->tr){ //saved backlog?
    object_process(nrtm->tr); // delete the previous(saved) object
    report_transaction(nrtm->tr, &log, nrtm->object_name, "NRTM:DEL:While deleting previous(saved) object");
    object_free(nrtm->tr->object);
    transaction_free(nrtm->tr); nrtm->tr=NULL;
  }
                                                                                                       
 ftime=time(NULL);
 obj_second1 = (float)(log.num_ok)/(ftime-stime);
 obj_second10 = (float)(log.num_ok+log.num_failed)/(ftime-stime);

  SQ_close_connection(sql_connection);
  g_string_free(g_line_buff, TRUE);
  fclose(ud_stream->stream);
  if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode)))unlink(tmpfile2);
  printf("\n\n******** report **********\n%d objects OK\n%d objects failed\n", log.num_ok, log.num_failed);
  fprintf(logfile,"\n******** report **********\n");
  fprintf(logfile," %d objects OK (%5.2f obj/s)\n", log.num_ok, obj_second1);
  fprintf(logfile," %d objects failed\n", log.num_failed);
  fprintf(logfile," average processing time %5.2f obj/s (%5.2f obj/min)\n", obj_second10, obj_second10*60);
  fclose(logfile);
  return(log.num_ok+log.num_failed);

} /* UD_process_stream */





/* [<][>][^][v][top][bottom][index][help] */