001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.IOException; 016import java.io.Serializable; 017import java.lang.annotation.Annotation; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.HashMap; 022import java.util.List; 023import java.util.Map; 024 025import org.eclipse.january.DatasetException; 026import org.eclipse.january.IMonitor; 027import org.eclipse.january.io.ILazyLoader; 028import org.eclipse.january.metadata.MetadataFactory; 029import org.eclipse.january.metadata.MetadataType; 030import org.eclipse.january.metadata.OriginMetadata; 031import org.eclipse.january.metadata.Reshapeable; 032import org.eclipse.january.metadata.Sliceable; 033import org.eclipse.january.metadata.Transposable; 034 035public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable { 036 private static final long serialVersionUID = 2467865859867440242L; 037 038 protected int[] oShape; // original shape 039 protected long size; // number of items 040 protected int dtype; // dataset type 041 protected int isize; // number of elements per item 042 043 protected ILazyLoader loader; 044 protected LazyDataset base = null; // used for transpose 045 046 // relative to loader or base 047 protected int prepShape = 0; // prepending and post-pending 048 protected int postShape = 0; // changes to shape 049 protected int[] begSlice = null; // slice begin 050 protected int[] delSlice = null; // slice delta 051 protected int[] map; // transposition map (same length as current shape) 052 protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null; 053 054 /** 055 * Create a lazy dataset 056 * @param name 057 * @param dtype dataset type 058 * @param elements 059 * @param shape 060 * @param loader 061 */ 062 public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) { 063 this.name = name; 064 this.shape = shape.clone(); 065 this.oShape = this.shape; 066 this.loader = loader; 067 this.dtype = dtype; 068 this.isize = elements; 069 try { 070 size = ShapeUtils.calcLongSize(shape); 071 } catch (IllegalArgumentException e) { 072 size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 073 } 074 } 075 076 /** 077 * Create a lazy dataset 078 * @param name 079 * @param dtype dataset type 080 * @param shape 081 * @param loader 082 */ 083 public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) { 084 this(name, dtype, 1, shape, loader); 085 } 086 087 /** 088 * Create a lazy dataset based on in-memory data (handy for testing) 089 * @param dataset 090 */ 091 public static LazyDataset createLazyDataset(final Dataset dataset) { 092 return new LazyDataset(dataset.getName(), dataset.getDType(), dataset.getElementsPerItem(), dataset.getShape(), 093 new ILazyLoader() { 094 private static final long serialVersionUID = -6725268922780517523L; 095 096 final Dataset d = dataset; 097 @Override 098 public boolean isFileReadable() { 099 return true; 100 } 101 102 @Override 103 public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException { 104 return d.getSlice(mon, slice); 105 } 106 }); 107 } 108 109 /** 110 * Can return -1 for unknown 111 */ 112 @Override 113 public int getDType() { 114 return dtype; 115 } 116 117 /** 118 * Can return -1 for unknown 119 */ 120 @Override 121 public int getElementsPerItem() { 122 return isize; 123 } 124 125 @Override 126 public int getSize() { 127 return (int) size; 128 } 129 130 @Override 131 public String toString() { 132 StringBuilder out = new StringBuilder(); 133 134 if (name != null && name.length() > 0) { 135 out.append("Lazy dataset '"); 136 out.append(name); 137 out.append("' has shape ["); 138 } else { 139 out.append("Lazy dataset shape is ["); 140 } 141 int rank = shape == null ? 0 : shape.length; 142 143 if (rank > 0 && shape[0] >= 0) { 144 out.append(shape[0]); 145 } 146 for (int i = 1; i < rank; i++) { 147 out.append(", " + shape[i]); 148 } 149 out.append(']'); 150 151 return out.toString(); 152 } 153 154 @Override 155 public boolean equals(Object obj) { 156 if (!super.equals(obj)) 157 return false; 158 159 LazyDataset other = (LazyDataset) obj; 160 if (dtype != other.dtype) { 161 return false; 162 } 163 if (isize != other.isize) { 164 return false; 165 } 166 167 if (!Arrays.equals(shape, other.shape)) { 168 return false; 169 } 170 171 if (loader != other.loader) { 172 return false; 173 } 174 175 if (prepShape != other.prepShape) { 176 return false; 177 } 178 179 if (postShape != other.postShape) { 180 return false; 181 } 182 183 if (!Arrays.equals(begSlice, other.begSlice)) { 184 return false; 185 } 186 if (!Arrays.equals(delSlice, other.delSlice)) { 187 return false; 188 } 189 if (!Arrays.equals(map, other.map)) { 190 return false; 191 } 192 return true; 193 } 194 195 @Override 196 public LazyDataset clone() { 197 LazyDataset ret = new LazyDataset(new String(name), dtype, isize, oShape, loader); 198 ret.shape = shape; 199 ret.size = size; 200 ret.prepShape = prepShape; 201 ret.postShape = postShape; 202 ret.begSlice = begSlice; 203 ret.delSlice = delSlice; 204 ret.map = map; 205 ret.base = base; 206 ret.metadata = copyMetadata(); 207 ret.oMetadata = oMetadata; 208 return ret; 209 } 210 211 @Override 212 public void setShape(int... shape) { 213 setShapeInternal(shape); 214 } 215 216 @Override 217 public LazyDataset squeezeEnds() { 218 setShapeInternal(ShapeUtils.squeezeShape(shape, true)); 219 return this; 220 } 221 222 @Override 223 public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException { 224 return getSlice(null, start, stop, step); 225 } 226 227 @Override 228 public Dataset getSlice(Slice... slice) throws DatasetException { 229 if (slice == null || slice.length == 0) { 230 return getSlice(null, new SliceND(shape)); 231 } 232 return getSlice(null, new SliceND(shape, slice)); 233 } 234 235 @Override 236 public Dataset getSlice(SliceND slice) throws DatasetException { 237 return getSlice(null, slice); 238 } 239 240 @Override 241 public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException { 242 if (slice == null || slice.length == 0) { 243 return getSlice(monitor, new SliceND(shape)); 244 } 245 return getSlice(monitor, new SliceND(shape, slice)); 246 } 247 248 @Override 249 public LazyDataset getSliceView(Slice... slice) { 250 if (slice == null || slice.length == 0) { 251 return getSliceView(new SliceND(shape)); 252 } 253 return getSliceView(new SliceND(shape, slice)); 254 } 255 256 /** 257 * @param nShape 258 */ 259 private void setShapeInternal(int... nShape) { 260 261 long nsize = ShapeUtils.calcLongSize(nShape); 262 if (nsize != size) { 263 throw new IllegalArgumentException("Size of new shape is not equal to current size"); 264 } 265 266 if (nsize == 1) { 267 shape = nShape.clone(); 268 return; 269 } 270 271 int ob = -1; // first non-unit dimension 272 int or = shape.length; 273 for (int i = 0; i < or; i++) { 274 if (shape[i] != 1) { 275 ob = i; 276 break; 277 } 278 } 279 assert ob >= 0; 280 int oe = -1; // last non-unit dimension 281 for (int i = or - 1; i >= ob; i--) { 282 if (shape[i] != 1) { 283 oe = i; 284 break; 285 } 286 } 287 assert oe >= 0; 288 oe++; 289 290 int nb = -1; // first non-unit dimension 291 int nr = nShape.length; 292 for (int i = 0; i < nr; i++) { 293 if (nShape[i] != 1) { 294 nb = i; 295 break; 296 } 297 } 298 299 int i = ob; 300 int j = nb; 301 if (begSlice == null) { 302 for (; i < oe && j < nr; i++, j++) { 303 if (shape[i] != nShape[j]) { 304 throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape"); 305 } 306 } 307 } else { 308 int[] nBegSlice = new int[nr]; 309 int[] nDelSlice = new int[nr]; 310 Arrays.fill(nDelSlice, 1); 311 for (; i < oe && j < nr; i++, j++) { 312 if (shape[i] != nShape[j]) { 313 throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape"); 314 } 315 nBegSlice[j] = begSlice[i]; 316 nDelSlice[j] = delSlice[i]; 317 } 318 319 begSlice = nBegSlice; 320 delSlice = nDelSlice; 321 } 322 prepShape += nb - ob; 323 postShape += nr - oe; 324 325 storeMetadata(metadata, Reshapeable.class); 326 metadata = copyMetadata(); 327 reshapeMetadata(shape, nShape); 328 shape = nShape; 329 } 330 331 @Override 332 public LazyDataset getSliceView(int[] start, int[] stop, int[] step) { 333 return getSliceView(new SliceND(shape, start, stop, step)); 334 } 335 336 @Override 337 public LazyDataset getSliceView(SliceND slice) { 338 LazyDataset view = clone(); 339 if (slice.isAll()) 340 return view; 341 342 int[] lstart = slice.getStart(); 343 int[] lstep = slice.getStep(); 344 final int rank = shape.length; 345 346 int[] nShape = slice.getShape(); 347 view.shape = nShape; 348 view.size = ShapeUtils.calcLongSize(nShape); 349 if (begSlice == null) { 350 view.begSlice = lstart.clone(); 351 view.delSlice = lstep.clone(); 352 } else { 353 view.begSlice = new int[rank]; 354 view.delSlice = new int[rank]; 355 for (int i = 0; i < rank; i++) { 356 view.begSlice[i] = begSlice[i] + lstart[i] * delSlice[i]; 357 view.delSlice[i] = delSlice[i] * lstep[i]; 358 } 359 } 360 view.storeMetadata(metadata, Sliceable.class); 361 362 view.sliceMetadata(true, slice); 363 return view; 364 } 365 366 @Override 367 public LazyDataset getTransposedView(int... axes) { 368 LazyDataset view = clone(); 369 370 // everything now is seen through a map 371 axes = checkPermutatedAxes(shape, axes); 372 if (axes == null) 373 return view; 374 375 int r = shape.length; 376 view.shape = new int[r]; 377 for (int i = 0; i < r; i++) { 378 view.shape[i] = shape[axes[i]]; 379 } 380 381 view.prepShape = 0; 382 view.postShape = 0; 383 view.begSlice = null; 384 view.delSlice = null; 385 view.map = axes; 386 view.base = this; 387 view.storeMetadata(metadata, Transposable.class); 388 view.transposeMetadata(axes); 389 return view; 390 } 391 392 @Override 393 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException { 394 return getSlice(monitor, new SliceND(shape, start, stop, step)); 395 } 396 397 @Override 398 public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException { 399 400 if (loader != null && !loader.isFileReadable()) { 401 return null; // TODO add interaction to use plot (or remote) server to load dataset 402 } 403 404 SliceND nslice = calcTrueSlice(slice); 405 406 Dataset a; 407 if (base != null) { 408 a = base.getSlice(monitor, nslice); 409 } else { 410 try { 411 a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice)); 412 } catch (IOException e) { 413 logger.error("Problem getting {}: {}", String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()), 414 Arrays.toString(slice.getStep()), loader), e); 415 throw new DatasetException(e); 416 } 417 a.setName(name + AbstractDataset.BLOCK_OPEN + nslice.toString() + AbstractDataset.BLOCK_CLOSE); 418 if (metadata != null && a instanceof LazyDatasetBase) { 419 LazyDatasetBase ba = (LazyDatasetBase) a; 420 ba.metadata = copyMetadata(metadata, oMetadata); 421 // metadata axis may be larger than data 422 if (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape()) { 423 ba.sliceMetadata(true, nslice); 424 } 425 } 426 } 427 if (map != null) { 428 a = a.getTransposedView(map); 429 } 430 if (slice != null) { 431 a.setShape(slice.getShape()); 432 } 433 a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice.convertToSlice(), oShape, null, name)); 434 435 return a; 436 } 437 438 // reverse transform 439 private int[] getOriginal(int[] values) { 440 if (values == null) 441 return null; 442 int r = values.length; 443 if (map == null || r < 2) 444 return values; 445 int[] ovalues = new int[r]; 446 for (int i = 0; i < r; i++) { 447 ovalues[map[i]] = values[i]; 448 } 449 return ovalues; 450 } 451 452 protected final SliceND calcTrueSlice(SliceND slice) { 453 if (slice == null) { 454 slice = new SliceND(shape); 455 } 456 int[] lstart = slice.getStart(); 457 int[] lstop = slice.getStop(); 458 int[] lstep = slice.getStep(); 459 460 int[] nstart; 461 int[] nstop; 462 int[] nstep; 463 464 int r = base == null ? oShape.length : base.shape.length; 465 nstart = new int[r]; 466 nstop = new int[r]; 467 nstep = new int[r]; 468 Arrays.fill(nstop, 1); 469 Arrays.fill(nstep, 1); 470 { 471 int i = 0; 472 int j = 0; 473 if (prepShape < 0) { // ignore entries from new slice 474 i = -prepShape; 475 } else if (prepShape > 0) { 476 j = prepShape; 477 } 478 if (begSlice == null) { 479 for (; i < r && j < shape.length; i++, j++) { 480 nstart[i] = lstart[j]; 481 nstop[i] = lstop[j]; 482 int d = lstep[j]; 483 if (d < 0 && nstop[i] < 0) { // need to wrap around further 484 int l = base == null ? oShape[j]: base.shape[j]; 485 nstop[i] -= l; 486 } 487 nstep[i] = d; 488 } 489 } else { 490 for (; i < r && j < shape.length; i++, j++) { 491 int b = begSlice[j]; 492 int d = delSlice[j]; 493 nstart[i] = b + lstart[j] * d; 494 nstop[i] = b + (lstop[j] - 1) * d + (d >= 0 ? 1 : -1); 495 if (d < 0 && nstop[i] < 0) { // need to wrap around further 496 int l = base == null ? oShape[j]: base.shape[j]; 497 nstop[i] -= l; 498 } 499 nstep[i] = lstep[j] * d; 500 } 501 } 502 if (map != null) { 503 nstart = getOriginal(nstart); 504 nstop = getOriginal(nstop); 505 nstep = getOriginal(nstep); 506 } 507 } 508 509 return createSlice(nstart, nstop, nstep); 510 } 511 512 protected final IDataset transformInput(IDataset data) { 513 if (map == null) 514 return data; 515 return data.getTransposedView(map); 516 } 517 518 protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) { 519 return new SliceND(base == null ? oShape : base.shape, nstart, nstop, nstep); 520 } 521 522 /** 523 * Store metadata items that has given annotation 524 * @param origMetadata 525 * @param aclazz 526 */ 527 private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) { 528 List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz); 529 if (mclazzes.size() == 0) 530 return; 531 532 if (oMetadata == null) { 533 oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>(); 534 } 535 for (Class<? extends MetadataType> mc : mclazzes) { 536 if (oMetadata.containsKey(mc)) 537 continue; // do not overwrite original 538 539 List<MetadataType> l = origMetadata.get(mc); 540 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 541 for (MetadataType m : l) { 542 nl.add(m.clone()); 543 } 544 oMetadata.put(mc, nl); 545 } 546 } 547 548 @SuppressWarnings("unchecked") 549 private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) { 550 List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>(); 551 if (metadata == null) 552 return mclazzes; 553 554 for (Class<? extends MetadataType> c : metadata.keySet()) { 555 boolean hasAnn = false; 556 for (MetadataType m : metadata.get(c)) { 557 if (m == null) 558 continue; 559 560 Class<? extends MetadataType> mc = m.getClass(); 561 do { // iterate over super-classes 562 for (Field f : mc.getDeclaredFields()) { 563 if (f.isAnnotationPresent(aclazz)) { 564 hasAnn = true; 565 break; 566 } 567 } 568 Class<?> sclazz = mc.getSuperclass(); 569 if (!MetadataType.class.isAssignableFrom(sclazz)) 570 break; 571 mc = (Class<? extends MetadataType>) sclazz; 572 } while (!hasAnn); 573 if (hasAnn) 574 break; 575 } 576 if (hasAnn) { 577 mclazzes.add(c); 578 } 579 } 580 return mclazzes; 581 } 582 583 /** 584 * Gets the maximum size of a slice of a dataset in a given dimension 585 * which should normally fit in memory. Note that it might be possible 586 * to get more in memory, this is a conservative estimate and seems to 587 * almost always work at the size returned; providing Xmx is less than 588 * the physical memory. 589 * 590 * To get more in memory increase -Xmx setting or use an expression 591 * which calls a rolling function (like rmean) instead of slicing directly 592 * to memory. 593 * 594 * @param lazySet 595 * @param dimension 596 * @return maximum size of dimension that can be sliced. 597 */ 598 public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) { 599 // size in bytes of each item 600 final double size = DTypeUtils.getItemBytes(DTypeUtils.getDTypeFromClass(lazySet.getElementClass()), lazySet.getElementsPerItem()); 601 602 // Max in bytes takes into account our minimum requirement 603 final double max = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory()); 604 605 // Firstly if the whole dataset it likely to fit in memory, then we allow it. 606 // Space specified in bytes per item available 607 final double space = max/lazySet.getSize(); 608 609 // If we have room for this whole dataset, then fine 610 int[] shape = lazySet.getShape(); 611 if (space >= size) 612 return shape[dimension]; 613 614 // Otherwise estimate what we can fit in, conservatively. 615 // First get size of one slice, see it that fits, if not, still return 1 616 double sizeOneSlice = size; // in bytes 617 for (int dim = 0; dim < shape.length; dim++) { 618 if (dim == dimension) 619 continue; 620 sizeOneSlice *= shape[dim]; 621 } 622 double avail = max / sizeOneSlice; 623 if (avail < 1) 624 return 1; 625 626 // We fudge this to leave some room 627 return (int) Math.floor(avail/4d); 628 } 629}