001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 protected static boolean catchExceptions; 050 051 static { 052 /** 053 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE 054 */ 055 try { 056 catchExceptions = Boolean.getBoolean("run.in.eclipse"); 057 } catch (SecurityException e) { 058 // set a default for when the security manager does not allow access to the requested key 059 catchExceptions = false; 060 } 061 } 062 063 transient private boolean dirty = true; // indicate dirty state of metadata 064 protected String name = ""; 065 066 /** 067 * The shape or dimensions of the dataset 068 */ 069 protected int[] shape; 070 071 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 072 073 /** 074 * @return type of dataset item 075 */ 076 abstract public int getDType(); 077 078 @Override 079 public Class<?> getElementClass() { 080 return DTypeUtils.getElementClass(getDType()); 081 } 082 083 @Override 084 public LazyDatasetBase clone() { 085 return null; 086 } 087 088 @Override 089 public boolean equals(Object obj) { 090 if (this == obj) { 091 return true; 092 } 093 if (obj == null) { 094 return false; 095 } 096 if (!getClass().equals(obj.getClass())) { 097 return false; 098 } 099 100 LazyDatasetBase other = (LazyDatasetBase) obj; 101 if (getDType() != other.getDType()) { 102 return false; 103 } 104 if (getElementsPerItem() != other.getElementsPerItem()) { 105 return false; 106 } 107 if (!Arrays.equals(shape, other.shape)) { 108 return false; 109 } 110 return true; 111 } 112 113 @Override 114 public int hashCode() { 115 int hash = getDType() * 17 + getElementsPerItem(); 116 int rank = shape.length; 117 for (int i = 0; i < rank; i++) { 118 hash = hash*17 + shape[i]; 119 } 120 return hash; 121 } 122 123 @Override 124 public String getName() { 125 return name; 126 } 127 128 @Override 129 public void setName(String name) { 130 this.name = name; 131 } 132 133 @Override 134 public int[] getShape() { 135 return shape.clone(); 136 } 137 138 @Override 139 public int getRank() { 140 return shape.length; 141 } 142 143 /** 144 * This method allows anything that dirties the dataset to clear various metadata values 145 * so that the other methods can work correctly. 146 * @since 2.1 147 */ 148 public void setDirty() { 149 dirty = true; 150 } 151 152 /** 153 * Find first sub-interface of (or class that directly implements) MetadataType 154 * @param clazz 155 * @return sub-interface 156 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 157 */ 158 @SuppressWarnings("unchecked") 159 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 160 if (clazz.equals(MetadataType.class)) { 161 throw new IllegalArgumentException("Cannot accept MetadataType"); 162 } 163 164 if (clazz.isInterface()) { 165 return clazz; 166 } 167 168 if (clazz.isAnonymousClass()) { // special case 169 Class<?> s = clazz.getSuperclass(); 170 if (!s.equals(Object.class)) { 171 // only use super class if it is not an anonymous class of an interface 172 clazz = (Class<? extends MetadataType>) s; 173 } 174 } 175 176 for (Class<?> c : clazz.getInterfaces()) { 177 if (c.equals(MetadataType.class)) { 178 if (clazz.isAnonymousClass()) { 179 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 180 } 181 return clazz; 182 } 183 if (MetadataType.class.isAssignableFrom(c)) { 184 return (Class<? extends MetadataType>) c; 185 } 186 } 187 188 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 189 if (c != null) { 190 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 191 } 192 193 logger.error("Somehow the search for metadata type interface ended in a bad place"); 194 assert false; // should not be able to get here!!! 195 return null; 196 } 197 198 @Override 199 public void setMetadata(MetadataType metadata) { 200 addMetadata(metadata, true); 201 } 202 203 @Override 204 public void addMetadata(MetadataType metadata) { 205 addMetadata(metadata, false); 206 } 207 208 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 209 if (metadata == null) 210 return; 211 212 if (this.metadata == null) { 213 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 214 } 215 216 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 217 if (!this.metadata.containsKey(clazz)) { 218 this.metadata.put(clazz, new ArrayList<MetadataType>()); 219 } else if (clear) { 220 this.metadata.get(clazz).clear(); 221 } 222 this.metadata.get(clazz).add(metadata); 223 224 // add for special case of sub-interfaces of IMetadata 225 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 226 clazz = IMetadata.class; 227 if (!this.metadata.containsKey(clazz)) { 228 this.metadata.put(clazz, new ArrayList<MetadataType>()); 229 } else if (clear) { 230 this.metadata.get(clazz).clear(); 231 } 232 this.metadata.get(clazz).add(metadata); 233 } 234 } 235 236 @Override 237 @Deprecated 238 public synchronized IMetadata getMetadata() { 239 return getFirstMetadata(IMetadata.class); 240 } 241 242 @SuppressWarnings("unchecked") 243 @Override 244 public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException { 245 if (metadata == null) { 246 dirty = false; 247 return null; 248 } 249 250 if (dirty) { 251 dirtyMetadata(); 252 dirty = false; 253 } 254 255 if (clazz == null) { 256 List<S> all = new ArrayList<S>(); 257 for (Class<? extends MetadataType> c : metadata.keySet()) { 258 all.addAll((Collection<S>) metadata.get(c)); 259 } 260 return all; 261 } 262 263 return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 264 } 265 266 @Override 267 public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) { 268 try { 269 List<S> ml = getMetadata(clazz); 270 if (ml == null) { 271 return null; 272 } 273 for (S t : ml) { 274 if (clazz.isInstance(t)) { 275 return t; 276 } 277 } 278 } catch (Exception e) { 279 logger.error("Get metadata failed!",e); 280 } 281 282 return null; 283 } 284 285 @Override 286 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 287 if (metadata == null) 288 return; 289 290 if (clazz == null) { 291 metadata.clear(); 292 return; 293 } 294 295 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 296 if( list != null) { 297 list.clear(); 298 } 299 } 300 301 /** 302 * @since 2.0 303 */ 304 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 305 return copyMetadata(metadata, null); 306 } 307 308 /** 309 * @since 2.0 310 */ 311 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 312 return copyMetadata(metadata, null); 313 } 314 315 /** 316 * Copy metadata. If oMetadata is not null, then copy from that when it has the corresponding items 317 * @since 2.1 318 * @param metadata 319 * @param oMetadata can be null 320 * @return 321 */ 322 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata, Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata) { 323 if (metadata == null) 324 return null; 325 326 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 327 328 for (Class<? extends MetadataType> c : metadata.keySet()) { 329 List<MetadataType> l = metadata.get(c); 330 if (!l.isEmpty() && oMetadata != null && oMetadata.containsKey(c)) { // only override when not empty 331 l = oMetadata.get(c); 332 } 333 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 334 map.put(c, nl); 335 for (MetadataType m : l) { 336 nl.add(m.clone()); 337 } 338 } 339 return map; 340 } 341 342 interface MetadatasetAnnotationOperation { 343 /** 344 * Process value of given field 345 * <p> 346 * When the field is not a container then the returned value 347 * may replace the old value 348 * @param f given field 349 * @param o value of field 350 * @return transformed field 351 */ 352 Object processField(Field f, Object o); 353 354 /** 355 * @return annotated class 356 */ 357 Class<? extends Annotation> getAnnClass(); 358 359 /** 360 * @param axis 361 * @return number of dimensions to insert or remove 362 */ 363 int change(int axis); 364 365 /** 366 * 367 * @return rank or -1 to match 368 */ 369 int getNewRank(); 370 371 /** 372 * Run on given lazy dataset 373 * @param lz 374 * @return 375 */ 376 ILazyDataset run(ILazyDataset lz); 377 } 378 379 class MdsSlice implements MetadatasetAnnotationOperation { 380 private boolean asView; 381 private SliceND slice; 382 private int[] oShape; 383 private long oSize; 384 385 public MdsSlice(boolean asView, SliceND slice) { 386 this.asView = asView; 387 this.slice = slice; 388 oShape = slice.getSourceShape(); 389 oSize = ShapeUtils.calcLongSize(oShape); 390 } 391 392 @Override 393 public Object processField(Field field, Object o) { 394 return o; 395 } 396 397 @Override 398 public Class<? extends Annotation> getAnnClass() { 399 return Sliceable.class; 400 } 401 402 @Override 403 public int change(int axis) { 404 return 0; 405 } 406 407 @Override 408 public int getNewRank() { 409 return -1; 410 } 411 412 @Override 413 public ILazyDataset run(ILazyDataset lz) { 414 int rank = lz.getRank(); 415 if (slice.getStart().length != rank) { 416 throw new IllegalArgumentException("Slice dimensions do not match dataset!"); 417 } 418 419 int[] shape = lz.getShape(); 420 SliceND nslice; 421 if (lz.getSize() == oSize) { 422 nslice = slice; 423 } else { 424 nslice = slice.clone(); 425 for (int i = 0; i < rank; i++) { 426 if (shape[i] >= oShape[i]) continue; 427 if (shape[i] == 1) { 428 nslice.setSlice(i, 0, 1, 1); 429 } else { 430 throw new IllegalArgumentException("Sliceable dataset has invalid size!"); 431 } 432 } 433 } 434 435 if (asView || (lz instanceof IDataset)) 436 return lz.getSliceView(nslice); 437 try { 438 return lz.getSlice(nslice); 439 } catch (DatasetException e) { 440 logger.error("Could not slice dataset in metadata", e); 441 return null; 442 } 443 } 444 } 445 446 class MdsReshape implements MetadatasetAnnotationOperation { 447 private boolean matchRank; 448 private int[] oldShape; 449 private int[] newShape; 450 boolean onesOnly; 451 int[] differences; 452 453 /* 454 * if only ones then record differences (insertions and deletions) 455 * 456 * if shape changing, find broadcasted dimensions and disallow 457 * merging that include those dimensions 458 */ 459 public MdsReshape(final int[] oldShape, final int[] newShape) { 460 this.oldShape = oldShape; 461 this.newShape = newShape; 462 differences = null; 463 } 464 465 @Override 466 public Object processField(Field field, Object o) { 467 Annotation a = field.getAnnotation(Reshapeable.class); 468 if (a != null) { // cannot be null 469 matchRank = ((Reshapeable) a).matchRank(); 470 } 471 return o; 472 } 473 474 @Override 475 public Class<? extends Annotation> getAnnClass() { 476 return Reshapeable.class; 477 } 478 479 @Override 480 public int change(int axis) { 481 if (matchRank) { 482 if (differences == null) 483 init(); 484 485 if (onesOnly) { 486 return differences[axis]; 487 } 488 throw new UnsupportedOperationException("TODO support other shape operations"); 489 } 490 return 0; 491 } 492 493 @Override 494 public int getNewRank() { 495 return matchRank ? newShape.length : -1; 496 } 497 498 private void init() { 499 int or = oldShape.length - 1; 500 int nr = newShape.length - 1; 501 if (or < 0 || nr < 0) { // zero-rank shapes 502 onesOnly = true; 503 differences = new int[1]; 504 differences[0] = or < 0 ? nr + 1 : or + 1; 505 return; 506 } 507 int ob = 0; 508 int nb = 0; 509 onesOnly = true; 510 do { 511 while (oldShape[ob] == 1 && ob < or) { 512 ob++; // next non-unit dimension 513 } 514 while (newShape[nb] == 1 && nb < nr) { 515 nb++; 516 } 517 if (oldShape[ob++] != newShape[nb++]) { 518 onesOnly = false; 519 break; 520 } 521 } while (ob <= or && nb <= nr); 522 523 ob = 0; 524 nb = 0; 525 differences = new int[or + 2]; 526 if (onesOnly) { 527 // work out unit dimensions removed from or add to old 528 int j = 0; 529 do { 530 if (oldShape[ob] != 1 && newShape[nb] != 1) { 531 ob++; 532 nb++; 533 } else { 534 while (oldShape[ob] == 1 && ob < or) { 535 ob++; 536 differences[j]--; 537 } 538 while (newShape[nb] == 1 && nb < nr) { 539 nb++; 540 differences[j]++; 541 } 542 } 543 j++; 544 } while (ob <= or && nb <= nr && j <= or); 545 while (ob <= or && oldShape[ob] == 1) { 546 ob++; 547 differences[j]--; 548 } 549 while (nb <= nr && newShape[nb] == 1) { 550 nb++; 551 differences[j]++; 552 } 553 } else { 554 if (matchRank) { 555 logger.error("Combining dimensions is currently not supported"); 556 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 557 } 558 // work out mapping: contiguous dimensions can be grouped or split 559 while (ob <= or && nb <= nr) { 560 int ol = oldShape[ob]; 561 while (ol == 1 && ol <= or) { 562 ob++; 563 ol = oldShape[ob]; 564 } 565 int oe = ob + 1; 566 int nl = newShape[nb]; 567 while (nl == 1 && nl <= nr) { 568 nb++; 569 nl = newShape[nb]; 570 } 571 int ne = nb + 1; 572 if (ol < nl) { 573 differences[ob] = 1; 574 do { // case where new shape combines several dimensions into one dimension 575 if (oe == (or + 1)) { 576 break; 577 } 578 differences[oe] = 1; 579 ol *= oldShape[oe++]; 580 } while (ol < nl); 581 differences[oe - 1] = oe - ob; // signal end with difference 582 if (nl != ol) { 583 logger.error("Single dimension is incompatible with subshape"); 584 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 585 } 586 } else if (ol > nl) { 587 do { // case where new shape spreads single dimension over several dimensions 588 if (ne == (nr + 1)) { 589 break; 590 } 591 nl *= newShape[ne++]; 592 } while (nl < ol); 593 if (nl != ol) { 594 logger.error("Subshape is incompatible with single dimension"); 595 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 596 } 597 598 } 599 600 ob = oe; 601 nb = ne; 602 } 603 604 } 605 } 606 607 @Override 608 public ILazyDataset run(ILazyDataset lz) { 609 if (differences == null) 610 init(); 611 612 int[] lshape = lz.getShape(); 613 if (Arrays.equals(newShape, lshape)) { 614 return lz; 615 } 616 int or = lz.getRank(); 617 int nr = newShape.length; 618 int[] nshape = new int[nr]; 619 Arrays.fill(nshape, 1); 620 if (onesOnly) { 621 // ignore omit removed dimensions 622 for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) { 623 int c = differences[i]; 624 if (c == 0) { 625 nshape[di++] = lshape[si++]; 626 } else if (c > 0) { 627 while (c-- > 0 && di < nr) { 628 di++; 629 } 630 } else if (c < 0) { 631 si -= c; // remove dimensions by skipping forward in source array 632 } 633 } 634 } else { 635 boolean[] broadcast = new boolean[or]; 636 for (int ob = 0; ob < or; ob++) { 637 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 638 } 639 int osize = lz.getSize(); 640 641 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 642 int ob = 0; 643 int nsize = 1; 644 for (int i = 0; i < nr; i++) { 645 if (ob < or && broadcast[ob]) { 646 if (differences[ob] != 0) { 647 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 648 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 649 } 650 } else { 651 nshape[i] = nsize < osize ? newShape[i] : 1; 652 } 653 nsize *= nshape[i]; 654 ob++; 655 } 656 } 657 658 ILazyDataset nlz = lz.getSliceView(); 659 if (lz instanceof Dataset) { 660 nlz = ((Dataset) lz).reshape(nshape); 661 } else { 662 nlz = lz.getSliceView(); 663 nlz.setShape(nshape); 664 } 665 return nlz; 666 } 667 } 668 669 class MdsTranspose implements MetadatasetAnnotationOperation { 670 int[] map; 671 672 public MdsTranspose(final int[] axesMap) { 673 map = axesMap; 674 } 675 676 @SuppressWarnings({ "rawtypes", "unchecked" }) 677 @Override 678 public Object processField(Field f, Object o) { 679 // reorder arrays and lists according the axes map 680 if (o.getClass().isArray()) { 681 int l = Array.getLength(o); 682 if (l == map.length) { 683 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 684 for (int i = 0; i < l; i++) { 685 Array.set(narray, i, Array.get(o, map[i])); 686 } 687 for (int i = 0; i < l; i++) { 688 Array.set(o, i, Array.get(narray, i)); 689 } 690 } 691 } else if (o instanceof List<?>) { 692 List list = (List) o; 693 int l = list.size(); 694 if (l == map.length) { 695 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 696 for (int i = 0; i < l; i++) { 697 Array.set(narray, i, list.get(map[i])); 698 } 699 list.clear(); 700 for (int i = 0; i < l; i++) { 701 list.add(Array.get(narray, i)); 702 } 703 } 704 } 705 return o; 706 } 707 708 @Override 709 public Class<? extends Annotation> getAnnClass() { 710 return Transposable.class; 711 } 712 713 @Override 714 public int change(int axis) { 715 return 0; 716 } 717 718 @Override 719 public int getNewRank() { 720 return -1; 721 } 722 723 @Override 724 public ILazyDataset run(ILazyDataset lz) { 725 return lz.getTransposedView(map); 726 } 727 } 728 729 class MdsDirty implements MetadatasetAnnotationOperation { 730 731 @Override 732 public Object processField(Field f, Object o) { 733 // throw exception if not boolean??? 734 Class<?> t = f.getType(); 735 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 736 if (o.equals(false)) { 737 o = true; 738 } 739 } 740 return o; 741 } 742 743 @Override 744 public Class<? extends Annotation> getAnnClass() { 745 return Dirtiable.class; 746 } 747 748 @Override 749 public int change(int axis) { 750 return 0; 751 } 752 753 @Override 754 public int getNewRank() { 755 return -1; 756 } 757 758 @Override 759 public ILazyDataset run(ILazyDataset lz) { 760 return lz; 761 } 762 } 763 764 /** 765 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 766 * dataset after cloning the metadata 767 * @param asView if true then just a view 768 * @param slice 769 */ 770 protected void sliceMetadata(boolean asView, final SliceND slice) { 771 processAnnotatedMetadata(new MdsSlice(asView, slice), true); 772 } 773 774 /** 775 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 776 * or setting the shape 777 * 778 * @param newShape 779 */ 780 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 781 processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true); 782 } 783 784 /** 785 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 786 * dataset after cloning the metadata 787 * @param axesMap 788 */ 789 protected void transposeMetadata(final int[] axesMap) { 790 processAnnotatedMetadata(new MdsTranspose(axesMap), true); 791 } 792 793 /** 794 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 795 * @since 2.0 796 */ 797 protected void dirtyMetadata() { 798 processAnnotatedMetadata(new MdsDirty(), true); 799 } 800 801 @SuppressWarnings("unchecked") 802 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) { 803 if (metadata == null) 804 return; 805 806 for (Class<? extends MetadataType> c : metadata.keySet()) { 807 for (MetadataType m : metadata.get(c)) { 808 if (m == null) 809 continue; 810 811 Class<? extends MetadataType> mc = m.getClass(); 812 do { // iterate over super-classes 813 processClass(op, m, mc, throwException); 814 Class<?> sclazz = mc.getSuperclass(); 815 if (!MetadataType.class.isAssignableFrom(sclazz)) 816 break; 817 mc = (Class<? extends MetadataType>) sclazz; 818 } while (true); 819 } 820 } 821 } 822 823 @SuppressWarnings({ "unchecked", "rawtypes" }) 824 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) { 825 for (Field f : mc.getDeclaredFields()) { 826 if (!f.isAnnotationPresent(op.getAnnClass())) 827 continue; 828 829 try { 830 f.setAccessible(true); 831 Object o = f.get(m); 832 if (o == null) 833 continue; 834 835 Object no = op.processField(f, o); 836 if (no != o) { 837 f.set(m, no); 838 continue; 839 } 840 Object r = null; 841 if (o instanceof ILazyDataset) { 842 try { 843 f.set(m, op.run((ILazyDataset) o)); 844 } catch (Exception e) { 845 logger.error("Problem processing " + o, e); 846 if (!catchExceptions) 847 throw e; 848 } 849 } else if (o.getClass().isArray()) { 850 int l = Array.getLength(o); 851 if (l <= 0) 852 continue; 853 854 for (int i = 0; r == null && i < l; i++) { 855 r = Array.get(o, i); 856 } 857 int n = op.getNewRank(); 858 if (r == null) { 859 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 860 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 861 } 862 continue; 863 } 864 if (n < 0) 865 n = l; 866 Object narray = Array.newInstance(r.getClass(), n); 867 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 868 int c = op.change(i); 869 if (c == 0) { 870 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 871 } else if (c > 0) { 872 di += c; // add nulls by skipping forward in destination array 873 } else if (c < 0) { 874 si -= c; // remove dimensions by skipping forward in source array 875 } 876 } 877 if (n == l) { 878 for (int i = 0; i < l; i++) { 879 Array.set(o, i, Array.get(narray, i)); 880 } 881 } else { 882 f.set(m, narray); 883 } 884 } else if (o instanceof List<?>) { 885 List list = (List) o; 886 int l = list.size(); 887 if (l <= 0) 888 continue; 889 890 for (int i = 0; r == null && i < l; i++) { 891 r = list.get(i); 892 } 893 int n = op.getNewRank(); 894 if (r == null) { 895 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 896 list.clear(); 897 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 898 list.add(null); 899 } 900 } 901 continue; 902 } 903 904 if (n < 0) 905 n = l; 906 Object narray = Array.newInstance(r.getClass(), n); 907 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 908 int c = op.change(i); 909 if (c == 0) { 910 Array.set(narray, di++, processObject(op, list.get(si++))); 911 } else if (c > 0) { 912 di += c; // add nulls by skipping forward in destination array 913 } else if (c < 0) { 914 si -= c; // remove dimensions by skipping forward in source array 915 } 916 } 917 list.clear(); 918 for (int i = 0; i < n; i++) { 919 list.add(Array.get(narray, i)); 920 } 921 } else if (o instanceof Map<?,?>) { 922 Map map = (Map) o; 923 for (Object k : map.keySet()) { 924 map.put(k, processObject(op, map.get(k))); 925 } 926 } 927 } catch (Exception e) { 928 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 929 if (throwException) 930 throw new RuntimeException(e); 931 } 932 } 933 } 934 935 @SuppressWarnings({ "unchecked", "rawtypes" }) 936 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 937 if (o == null) 938 return o; 939 940 if (o instanceof ILazyDataset) { 941 try { 942 return op.run((ILazyDataset) o); 943 } catch (Exception e) { 944 logger.error("Problem processing " + o, e); 945 if (!catchExceptions) 946 throw e; 947 } 948 } else if (o.getClass().isArray()) { 949 int l = Array.getLength(o); 950 for (int i = 0; i < l; i++) { 951 Array.set(o, i, processObject(op, Array.get(o, i))); 952 } 953 } else if (o instanceof List<?>) { 954 List list = (List) o; 955 for (int i = 0, imax = list.size(); i < imax; i++) { 956 list.set(i, processObject(op, list.get(i))); 957 } 958 } else if (o instanceof Map<?,?>) { 959 Map map = (Map) o; 960 for (Object k : map.keySet()) { 961 map.put(k, processObject(op, map.get(k))); 962 } 963 } 964 return o; 965 } 966 967 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 968 for (Class<? extends MetadataType> mc : oldMetadata.keySet()) { 969 metadata.put(mc, oldMetadata.get(mc)); 970 } 971 } 972 973 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 974 ILazyDataset d = null; 975 if (blob instanceof ILazyDataset) { 976 d = (ILazyDataset) blob; 977 if (d instanceof IDataset) { 978 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 979 int is = ed.getElementsPerItem(); 980 if (is != 1 && is != getElementsPerItem()) { 981 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 982 } 983 d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 984 } else if (!keepLazy) { 985 final int is = getElementsPerItem(); 986 try { 987 d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 988 } catch (DatasetException e) { 989 logger.error("Could not get data from lazy dataset", e); 990 return null; 991 } 992 } 993 } else { 994 final int is = getElementsPerItem(); 995 if (is == 1) { 996 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 997 } else { 998 try { 999 d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob); 1000 } catch (IllegalArgumentException e) { // if only single value supplied try again 1001 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1002 } 1003 } 1004 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 1005 d.setShape(shape.clone()); 1006 } 1007 } 1008 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 1009 d.setShape(s.get(0)); 1010 1011 return d; 1012 } 1013 1014 @Override 1015 public void setErrors(Serializable errors) { 1016 if (shape == null) { 1017 throw new IllegalArgumentException("Cannot set errors for null dataset"); 1018 } 1019 if (errors == null) { 1020 clearMetadata(ErrorMetadata.class); 1021 return; 1022 } 1023 if (errors == this) { 1024 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 1025 return; 1026 } 1027 1028 ILazyDataset errorData = createFromSerializable(errors, true); 1029 1030 ErrorMetadata emd = getErrorMetadata(); 1031 if (emd == null) { 1032 try { 1033 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1034 setMetadata(emd); 1035 } catch (MetadataException me) { 1036 logger.error("Could not create metadata", me); 1037 } 1038 } 1039 emd.setError(errorData); 1040 } 1041 1042 protected ErrorMetadata getErrorMetadata() { 1043 try { 1044 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1045 if (el != null && !el.isEmpty()) { 1046 return el.get(0); 1047 } 1048 } catch (Exception e) { 1049 } 1050 return null; 1051 } 1052 1053 @Override 1054 public ILazyDataset getErrors() { 1055 ErrorMetadata emd = getErrorMetadata(); 1056 return emd == null ? null : emd.getError(); 1057 } 1058 1059 @Override 1060 public boolean hasErrors() { 1061 return LazyDatasetBase.this.getErrors() != null; 1062 } 1063 1064 /** 1065 * Check permutation axes 1066 * @param shape 1067 * @param axes 1068 * @return cleaned up axes or null if trivial 1069 */ 1070 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1071 int rank = shape == null ? 0 : shape.length; 1072 1073 if (axes == null || axes.length == 0) { 1074 axes = new int[rank]; 1075 for (int i = 0; i < rank; i++) { 1076 axes[i] = rank - 1 - i; 1077 } 1078 } 1079 1080 if (axes.length != rank) { 1081 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1082 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1083 } 1084 1085 // check all permutation values are within bounds 1086 for (int i = 0; i < rank; i++) { 1087 axes[i] = ShapeUtils.checkAxis(rank, axes[i]); 1088 } 1089 1090 // check for a valid permutation (is this an unnecessary restriction?) 1091 int[] perm = axes.clone(); 1092 Arrays.sort(perm); 1093 1094 for (int i = 0; i < rank; i++) { 1095 if (perm[i] != i) { 1096 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1097 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1098 } 1099 } 1100 1101 if (Arrays.equals(axes, perm)) 1102 return null; // signal identity or trivial permutation 1103 1104 return axes; 1105 } 1106}