VBLADE/vblade-master/contrib/vblade-17-aio.2.diff

   1 diff -uprN vblade-17.orig/aoe.c vblade-17/aoe.c
   2 --- vblade-17.orig/aoe.c        2008-06-09 10:53:07.000000000 -0400
   3 +++ vblade-17/aoe.c     2008-06-09 11:05:23.000000000 -0400
   4 @@ -8,6 +8,9 @@
   5  #include <sys/stat.h>
   6  #include <fcntl.h>
   7  #include <netinet/in.h>
   8 +#include <errno.h>
   9 +#include <aio.h>
  10 +#include <poll.h>
  11  #include "dat.h"
  12  #include "fns.h"
  13
  14 @@ -22,6 +25,11 @@ char config[Nconfig];
  15  int nconfig = 0;
  16  int maxscnt = 2;
  17  char *ifname;
  18 +int queuepipe[2];
  19 +int pktlen[Nplaces], pending[Nplaces];
  20 +Ata *pkt[Nplaces];
  21 +Ataregs regs[Nplaces];
  22 +struct aiocb aiocb[Nplaces];
  23
  24  void
  25  aoead(int fd)                  // advertise the virtual blade
  26 @@ -78,32 +86,52 @@ getlba(uchar *p)
  27  }
  28
  29  int
  30 -aoeata(Ata *p, int pktlen)     // do ATA reqeust
  31 +aoeata(int place)      // do ATA reqeust
  32  {
  33 -       Ataregs r;
  34 -       int len = 60;
  35         int n;
  36 +       int len = 60; // minimum ethernet packet size
  37
  38 -       r.lba = getlba(p->lba);
  39 -       r.sectors = p->sectors;
  40 -       r.feature = p->err;
  41 -       r.cmd = p->cmd;
  42 -       if (atacmd(&r, (uchar *)(p+1), maxscnt*512, pktlen - sizeof(*p)) < 0) {
  43 -               p->h.flags |= Error;
  44 -               p->h.error = BadArg;
  45 +       regs[place].lba = getlba(pkt[place]->lba);
  46 +       regs[place].sectors = pkt[place]->sectors;
  47 +       regs[place].feature = pkt[place]->err;
  48 +       regs[place].cmd = pkt[place]->cmd;
  49 +       n = atacmd(regs + place, (uchar *)(pkt[place] + 1), maxscnt*512,
  50 +                               pktlen[place] - sizeof(Ata), aiocb + place);
  51 +       if (n < 0) {
  52 +               pkt[place]->h.flags |= Error;
  53 +               pkt[place]->h.error = BadArg;
  54                 return len;
  55 +       } else if (n > 0) {
  56 +               pending[place] = 1;
  57 +               return 0;
  58 +       }
  59 +       if (!(pkt[place]->aflag & Write) && (n = pkt[place]->sectors)) {
  60 +               n -= regs[place].sectors;
  61 +               len = sizeof (Ata) + (n*512);
  62         }
  63 -       if (!(p->aflag & Write))
  64 -       if ((n = p->sectors)) {
  65 -               n -= r.sectors;
  66 +       pkt[place]->sectors = regs[place].sectors;
  67 +       pkt[place]->err = regs[place].err;
  68 +       pkt[place]->cmd = regs[place].status;
  69 +       return len;
  70 +}
  71 +
  72 +int aoeatacomplete(int place, int pktlen)
  73 +{
  74 +       int n;
  75 +       int len = 60; // minimum ethernet packet size
  76 +       atacmdcomplete(regs + place, aiocb + place);
  77 +       if (!(pkt[place]->aflag & Write) && (n = pkt[place]->sectors)) {
  78 +               n -= regs[place].sectors;
  79                 len = sizeof (Ata) + (n*512);
  80         }
  81 -       p->sectors = r.sectors;
  82 -       p->err = r.err;
  83 -       p->cmd = r.status;
  84 +       pkt[place]->sectors = regs[place].sectors;
  85 +       pkt[place]->err = regs[place].err;
  86 +       pkt[place]->cmd = regs[place].status;
  87 +       pending[place] = 0;
  88         return len;
  89  }
  90
  91 +
  92  #define QCMD(x) ((x)->vercmd & 0xf)
  93
  94  // yes, this makes unnecessary copies.
  95 @@ -156,8 +184,9 @@ confcmd(Conf *p, int payload)       // process
  96  }
  97
  98  void
  99 -doaoe(Aoehdr *p, int n)
 100 +doaoe(int place)
 101  {
 102 +       Aoehdr *p = (Aoehdr *) pkt[place];
 103         int len;
 104         enum {  // config query header size
 105                 CHDR_SIZ = sizeof(Conf) - sizeof(((Conf *)0)->data),
 106 @@ -165,14 +194,16 @@ doaoe(Aoehdr *p, int n)
 107
 108         switch (p->cmd) {
 109         case ATAcmd:
 110 -               if (n < sizeof(Ata))
 111 +               if (pktlen[place] < sizeof(Ata))
 112 +                       return;
 113 +               len = aoeata(place);
 114 +               if (len == 0)
 115                         return;
 116 -               len = aoeata((Ata*)p, n);
 117                 break;
 118         case Config:
 119 -               if (n < CHDR_SIZ)
 120 +               if (pktlen[place] < CHDR_SIZ)
 121                         return;
 122 -               len = confcmd((Conf *)p, n - CHDR_SIZ);
 123 +               len = confcmd((Conf *)p, pktlen[place] - CHDR_SIZ);
 124                 if (len == 0)
 125                         return;
 126                 break;
 127 @@ -193,25 +224,129 @@ doaoe(Aoehdr *p, int n)
 128  }
 129
 130  void
 131 +doaoecomplete(int place)
 132 +{
 133 +       Aoehdr *p = (Aoehdr *) pkt[place];
 134 +       int len = aoeatacomplete(place, pktlen[place]);
 135 +       memmove(p->dst, p->src, 6);
 136 +       memmove(p->src, mac, 6);
 137 +       p->maj = htons(shelf);
 138 +       p->min = slot;
 139 +       p->flags |= Resp;
 140 +       if (putpkt(sfd, (uchar *) p, len) == -1) {
 141 +               perror("write to network");
 142 +               exit(1);
 143 +       }
 144 +
 145 +}
 146 +
 147 +// allocate the buffer so that the ata data area
 148 +// is page aligned for o_direct on linux
 149 +
 150 +void *
 151 +bufalloc(void **buf, long len)
 152 +{
 153 +       long psize;
 154 +       unsigned long n;
 155 +
 156 +       psize = sysconf(_SC_PAGESIZE);
 157 +       if (psize == -1) {
 158 +               perror("sysconf");
 159 +               exit(EXIT_FAILURE);
 160 +       }
 161 +       n = len/psize + 3;
 162 +       *buf = malloc(psize * n);
 163 +       if (!*buf) {
 164 +               perror("malloc");
 165 +               exit(EXIT_FAILURE);
 166 +       }
 167 +       n = (unsigned long) *buf;
 168 +       n += psize * 2;
 169 +       n &= ~(psize - 1);
 170 +       return (void *) (n - sizeof (Ata));
 171 +}
 172 +
 173 +void
 174 +sigio(int signo)
 175 +{
 176 +       const char dummy = 0;
 177 +       write(queuepipe[1], &dummy, 1);
 178 +}
 179 +
 180 +void
 181  aoe(void)
 182  {
 183         Aoehdr *p;
 184 -       uchar *buf;
 185 -       int n, sh;
 186 +       char dummy;
 187 +       int n, place, sh;
 188         enum { bufsz = 1<<16, };
 189 -
 190 -       buf = malloc(bufsz);
 191 +       sigset_t mask, oldmask;
 192 +       struct sigaction sigact;
 193 +       struct pollfd pollfds[2];
 194 +       void *freeme[Nplaces];
 195 +
 196 +       for (n = 0; n < Nplaces; n++) {
 197 +               pkt[n] = bufalloc(freeme + n, bufsz);
 198 +               pending[n] = 0;
 199 +       }
 200         aoead(sfd);
 201
 202 +       pipe(queuepipe);
 203 +       fcntl(queuepipe[0], F_SETFL, O_NONBLOCK);
 204 +       fcntl(queuepipe[1], F_SETFL, O_NONBLOCK);
 205 +
 206 +       sigemptyset(&sigact.sa_mask);
 207 +       sigact.sa_flags = 0;
 208 +       sigact.sa_sigaction = (void *) sigio;
 209 +       sigaction(SIGIO, &sigact, NULL);
 210 +
 211 +       sigemptyset(&mask);
 212 +       sigaddset(&mask, SIGIO);
 213 +       sigprocmask(SIG_BLOCK, &mask, &oldmask);
 214 +
 215 +       pollfds[0].fd = queuepipe[0];
 216 +       pollfds[1].fd = sfd;
 217 +       pollfds[0].events = pollfds[1].events = POLLIN;
 218 +
 219         for (;;) {
 220 -               n = getpkt(sfd, buf, bufsz);
 221 -               if (n < 0) {
 222 +               sigprocmask(SIG_SETMASK, &oldmask, NULL);
 223 +               n = poll(pollfds, 2, 1000);
 224 +               sigprocmask(SIG_BLOCK, &mask, NULL);
 225 +
 226 +               if (n < 0 && errno != EINTR) {
 227 +                       perror("poll");
 228 +                       continue;
 229 +               } else if (n == 0 || pollfds[0].revents & POLLIN) {
 230 +                       while(read(queuepipe[0], &dummy, 1) > 0);
 231 +                       for (place = 0; place < Nplaces; place++) {
 232 +                               if (!pending[place])
 233 +                                       continue;
 234 +                               if (aio_error(aiocb + place) == EINPROGRESS)
 235 +                                       continue;
 236 +                               doaoecomplete(place);
 237 +                               pollfds[1].events = POLLIN;
 238 +                       }
 239 +               }
 240 +
 241 +               if ((pollfds[1].revents & POLLIN) == 0)
 242 +                       continue;
 243 +
 244 +               for (place = 0; pending[place] && place < Nplaces; place++);
 245 +               if (place >= Nplaces) {
 246 +                       pollfds[1].events = 0;
 247 +                       continue;
 248 +               }
 249 +
 250 +               pktlen[place] = getpkt(sfd, (uchar *) pkt[place], bufsz);
 251 +               if (pktlen[place] < 0) {
 252 +                       if (errno == EINTR)
 253 +                               continue;
 254                         perror("read network");
 255                         exit(1);
 256                 }
 257 -               if (n < sizeof(Aoehdr))
 258 +               if (pktlen[place] < sizeof(Aoehdr))
 259                         continue;
 260 -               p = (Aoehdr *) buf;
 261 +               p = (Aoehdr *) pkt[place];
 262                 if (ntohs(p->type) != 0x88a2)
 263                         continue;
 264                 if (p->flags & Resp)
 265 @@ -223,9 +358,10 @@ aoe(void)
 266                         continue;
 267                 if (nmasks && !maskok(p->src))
 268                         continue;
 269 -               doaoe(p, n);
 270 +               doaoe(place);
 271         }
 272 -       free(buf);
 273 +       for (place = 0; place < Nplaces; place++)
 274 +               free(freeme[place]);
 275  }
 276
 277  void
 278 @@ -317,7 +453,7 @@ main(int argc, char **argv)
 279         }
 280         if (s.st_mode & (S_IWUSR|S_IWGRP|S_IWOTH))
 281                 omode = O_RDWR;
 282 -       bfd = open(argv[3], omode);
 283 +       bfd = opendisk(argv[3], omode);
 284         if (bfd == -1) {
 285                 perror("open");
 286                 exit(1);
 287 diff -uprN vblade-17.orig/ata.c vblade-17/ata.c
 288 --- vblade-17.orig/ata.c        2008-06-09 10:53:07.000000000 -0400
 289 +++ vblade-17/ata.c     2008-06-09 11:05:23.000000000 -0400
 290 @@ -3,6 +3,8 @@
 291  #include <string.h>
 292  #include <stdio.h>
 293  #include <sys/types.h>
 294 +#include <errno.h>
 295 +#include <aio.h>
 296  #include "dat.h"
 297  #include "fns.h"
 298
 299 @@ -98,7 +100,7 @@ atainit(void)
 300   * check for that.
 301   */
 302  int
 303 -atacmd(Ataregs *p, uchar *dp, int ndp, int payload) // do the ata cmd
 304 +atacmd(Ataregs *p, uchar *dp, int ndp, int payload, struct aiocb *aiocb) // do the ata cmd
 305  {
 306         vlong lba;
 307         ushort *ip;
 308 @@ -155,14 +157,29 @@ atacmd(Ataregs *p, uchar *dp, int ndp, i
 309                 return 0;
 310         }
 311         if (p->cmd == 0x20 || p->cmd == 0x24)
 312 -               n = getsec(bfd, dp, lba, p->sectors);
 313 +               n = getsec(bfd, dp, lba, p->sectors, aiocb);
 314         else {
 315                 // packet should be big enough to contain the data
 316                 if (payload < 512 * p->sectors)
 317                         return -1;
 318 -               n = putsec(bfd, dp, lba, p->sectors);
 319 +               n = putsec(bfd, dp, lba, p->sectors, aiocb);
 320         }
 321 -       n /= 512;
 322 +       if (n < 0) {
 323 +               p->err = ABRT;
 324 +               p->status = ERR|DRDY;
 325 +               p->lba += n;
 326 +               p->sectors -= n;
 327 +               return 0;
 328 +       }
 329 +       return 1; // callback expected
 330 +}
 331 +
 332 +
 333 +int
 334 +atacmdcomplete(Ataregs *p, struct aiocb *aiocb) // complete the ata cmd
 335 +{
 336 +       int n;
 337 +       n = aio_return(aiocb) / 512;
 338         if (n != p->sectors) {
 339                 p->err = ABRT;
 340                 p->status = ERR;
 341 @@ -173,4 +190,3 @@ atacmd(Ataregs *p, uchar *dp, int ndp, i
 342         p->sectors -= n;
 343         return 0;
 344  }
 345 -
 346 diff -uprN vblade-17.orig/dat.h vblade-17/dat.h
 347 --- vblade-17.orig/dat.h        2008-06-09 10:53:07.000000000 -0400
 348 +++ vblade-17/dat.h     2008-06-09 11:05:23.000000000 -0400
 349 @@ -111,6 +111,8 @@ enum {
 350         Nconfig = 1024,
 351
 352         Bufcount = 16,
 353 +
 354 +       Nplaces = 32,
 355  };
 356
 357  int    shelf, slot;
 358 diff -uprN vblade-17.orig/fns.h vblade-17/fns.h
 359 --- vblade-17.orig/fns.h        2008-06-09 10:53:07.000000000 -0400
 360 +++ vblade-17/fns.h     2008-06-09 11:07:21.000000000 -0400
 361 @@ -15,7 +15,8 @@ int   maskok(uchar *);
 362  // ata.c
 363
 364  void   atainit(void);
 365 -int    atacmd(Ataregs *, uchar *, int, int);
 366 +int    atacmd(Ataregs *, uchar *, int, int, struct aiocb *);
 367 +int    atacmdcomplete(Ataregs *, struct aiocb *);
 368
 369  // bpf.c
 370
 371 @@ -26,8 +27,9 @@ void  free_bpf_program(void *);
 372
 373  int    dial(char *);
 374  int    getea(int, char *, uchar *);
 375 -int    putsec(int, uchar *, vlong, int);
 376 -int    getsec(int, uchar *, vlong, int);
 377 +int    opendisk(const char *, int);
 378 +int    putsec(int, uchar *, vlong, int, struct aiocb *);
 379 +int    getsec(int, uchar *, vlong, int, struct aiocb *);
 380  int    putpkt(int, uchar *, int);
 381  int    getpkt(int, uchar *, int);
 382  vlong  getsize(int);
 383 diff -uprN vblade-17.orig/freebsd.c vblade-17/freebsd.c
 384 --- vblade-17.orig/freebsd.c    2008-06-09 10:53:07.000000000 -0400
 385 +++ vblade-17/freebsd.c 2008-06-09 11:05:23.000000000 -0400
 386 @@ -209,19 +209,40 @@ getea(int s, char *eth, uchar *ea)
 387         return(0);
 388  }
 389
 390 -
 391  int
 392 -getsec(int fd, uchar *place, vlong lba, int nsec)
 393 +opendisk(const char *disk, int omode)
 394  {
 395 -       return pread(fd, place, nsec * 512, lba * 512);
 396 +       return open(disk, omode);
 397  }
 398
 399  int
 400 -putsec(int fd, uchar *place, vlong lba, int nsec)
 401 -{
 402 -       return pwrite(fd, place, nsec * 512, lba * 512);
 403 +getsec(int fd, uchar *place, vlong lba, int nsec, struct aiocb *aiocb)
 404 +{
 405 +        bzero((char *) aiocb, sizeof(struct aiocb));
 406 +        aiocb->aio_fildes = fd;
 407 +        aiocb->aio_buf = place;
 408 +        aiocb->aio_nbytes = nsec * 512;
 409 +        aiocb->aio_offset = lba * 512;
 410 +        aiocb->aio_sigevent.sigev_notify = SIGEV_SIGNAL;
 411 +        aiocb->aio_sigevent.sigev_signo = SIGIO;
 412 +        aiocb->aio_sigevent.sigev_value.sival_ptr = aiocb;
 413 +        return aio_read(aiocb);
 414  }
 415
 416 +int
 417 +putsec(int fd, uchar *place, vlong lba, int nsec, struct aiocb *aiocb)
 418 +{
 419 +        bzero((char *) aiocb, sizeof(struct aiocb));
 420 +        aiocb->aio_fildes = fd;
 421 +        aiocb->aio_buf = place;
 422 +        aiocb->aio_nbytes = nsec * 512;
 423 +        aiocb->aio_offset = lba * 512;
 424 +        aiocb->aio_sigevent.sigev_notify = SIGEV_SIGNAL;
 425 +        aiocb->aio_sigevent.sigev_signo = SIGIO;
 426 +        aiocb->aio_sigevent.sigev_value.sival_ptr = aiocb;
 427 +        return aio_write(aiocb);
 428 +}
 429 +
 430  static int pktn = 0;
 431  static uchar *pktbp = NULL;
 432
 433 diff -uprN vblade-17.orig/linux.c vblade-17/linux.c
 434 --- vblade-17.orig/linux.c      2008-06-09 10:53:07.000000000 -0400
 435 +++ vblade-17/linux.c   2008-06-09 11:05:23.000000000 -0400
 436 @@ -1,5 +1,6 @@
 437  // linux.c: low level access routines for Linux
 438  #include "config.h"
 439 +#define _GNU_SOURCE
 440  #include <sys/socket.h>
 441  #include <stdio.h>
 442  #include <string.h>
 443 @@ -22,6 +23,9 @@
 444  #include <netinet/in.h>
 445  #include <linux/fs.h>
 446  #include <sys/stat.h>
 447 +#include <fcntl.h>
 448 +#include <errno.h>
 449 +#include <aio.h>
 450
 451  #include "dat.h"
 452  #include "fns.h"
 453 @@ -29,8 +33,6 @@
 454  int    getindx(int, char *);
 455  int    getea(int, char *, uchar *);
 456
 457 -
 458 -
 459  int
 460  dial(char *eth)                // get us a raw connection to an interface
 461  {
 462 @@ -84,7 +86,7 @@ getea(int s, char *name, uchar *ea)
 463         struct ifreq xx;
 464         int n;
 465
 466 -        strcpy(xx.ifr_name, name);
 467 +       strcpy(xx.ifr_name, name);
 468         n = ioctl(s, SIOCGIFHWADDR, &xx);
 469         if (n == -1) {
 470                 perror("Can't get hw addr");
 471 @@ -110,17 +112,37 @@ getmtu(int s, char *name)
 472  }
 473
 474  int
 475 -getsec(int fd, uchar *place, vlong lba, int nsec)
 476 +opendisk(const char *disk, int omode)
 477 +{
 478 +       return open(disk, omode|O_DIRECT);
 479 +}
 480 +
 481 +int
 482 +getsec(int fd, uchar *place, vlong lba, int nsec, struct aiocb *aiocb)
 483  {
 484 -       lseek(fd, lba * 512, 0);
 485 -       return read(fd, place, nsec * 512);
 486 +       bzero((char *) aiocb, sizeof(struct aiocb));
 487 +       aiocb->aio_fildes = fd;
 488 +       aiocb->aio_buf = place;
 489 +       aiocb->aio_nbytes = nsec * 512;
 490 +       aiocb->aio_offset = lba * 512;
 491 +       aiocb->aio_sigevent.sigev_notify = SIGEV_SIGNAL;
 492 +       aiocb->aio_sigevent.sigev_signo = SIGIO;
 493 +       aiocb->aio_sigevent.sigev_value.sival_ptr = aiocb;
 494 +       return aio_read(aiocb);
 495  }
 496
 497  int
 498 -putsec(int fd, uchar *place, vlong lba, int nsec)
 499 +putsec(int fd, uchar *place, vlong lba, int nsec, struct aiocb *aiocb)
 500  {
 501 -       lseek(fd, lba * 512, 0);
 502 -       return write(fd, place, nsec * 512);
 503 +       bzero((char *) aiocb, sizeof(struct aiocb));
 504 +       aiocb->aio_fildes = fd;
 505 +       aiocb->aio_buf = place;
 506 +       aiocb->aio_nbytes = nsec * 512;
 507 +       aiocb->aio_offset = lba * 512;
 508 +       aiocb->aio_sigevent.sigev_notify = SIGEV_SIGNAL;
 509 +       aiocb->aio_sigevent.sigev_signo = SIGIO;
 510 +       aiocb->aio_sigevent.sigev_value.sival_ptr = aiocb;
 511 +       return aio_write(aiocb);
 512  }
 513
 514  int
 515 diff -uprN vblade-17.orig/linux.h vblade-17/linux.h
 516 --- vblade-17.orig/linux.h      2008-06-09 10:53:07.000000000 -0400
 517 +++ vblade-17/linux.h   2008-06-09 11:05:23.000000000 -0400
 518 @@ -6,6 +6,6 @@ typedef long long vlong;
 519  int    dial(char *);
 520  int    getindx(int, char *);
 521  int    getea(int, char *, uchar *);
 522 -int    getsec(int, uchar *, vlong, int);
 523 -int    putsec(int, uchar *, vlong, int);
 524 +int    getsec(int, uchar *, vlong, int, struct aiocb *);
 525 +int    putsec(int, uchar *, vlong, int, struct aiocb *);
 526  vlong  getsize(int);
 527 diff -uprN vblade-17.orig/makefile vblade-17/makefile
 528 --- vblade-17.orig/makefile     2008-06-09 10:53:07.000000000 -0400
 529 +++ vblade-17/makefile  2008-06-09 11:05:23.000000000 -0400
 530 @@ -13,7 +13,7 @@ CFLAGS += -Wall -g -O2
 531  CC = gcc
 532
 533  vblade: $O
 534 -       ${CC} -o vblade $O
 535 +       ${CC} -lrt -o vblade $O
 536
 537  aoe.o : aoe.c config.h dat.h fns.h makefile
 538         ${CC} ${CFLAGS} -c $<