001 /*
002 // $Id: //open/mondrian/src/main/mondrian/olap/fun/AggregateFunDef.java#23 $
003 // This software is subject to the terms of the Common Public License
004 // Agreement, available at the following URL:
005 // http://www.opensource.org/licenses/cpl.html.
006 // Copyright (C) 2005-2008 Julian Hyde
007 // All Rights Reserved.
008 // You must accept the terms of that agreement to use this software.
009 */
010 package mondrian.olap.fun;
011
012 import mondrian.calc.Calc;
013 import mondrian.calc.ExpCompiler;
014 import mondrian.calc.ListCalc;
015 import mondrian.calc.impl.AbstractDoubleCalc;
016 import mondrian.calc.impl.GenericCalc;
017 import mondrian.calc.impl.ValueCalc;
018 import mondrian.mdx.ResolvedFunCall;
019 import mondrian.olap.*;
020 import mondrian.rolap.RolapAggregator;
021 import mondrian.rolap.RolapEvaluator;
022
023 import java.util.*;
024
025 import org.eigenbase.util.property.IntegerProperty;
026
027 /**
028 * Definition of the <code>AGGREGATE</code> MDX function.
029 *
030 * @author jhyde
031 * @since 2005/8/14
032 * @version $Id: //open/mondrian/src/main/mondrian/olap/fun/AggregateFunDef.java#23 $
033 */
034 public class AggregateFunDef extends AbstractAggregateFunDef {
035 static final ReflectiveMultiResolver resolver =
036 new ReflectiveMultiResolver(
037 "Aggregate", "Aggregate(<Set>[, <Numeric Expression>])",
038 "Returns a calculated value using the appropriate aggregate function, based on the context of the query.",
039 new String[]{"fnx", "fnxn"},
040 AggregateFunDef.class);
041
042 public AggregateFunDef(FunDef dummyFunDef) {
043 super(dummyFunDef);
044 }
045
046 public Calc compileCall(ResolvedFunCall call, ExpCompiler compiler) {
047 final ListCalc listCalc = compiler.compileList(call.getArg(0));
048 final Calc calc = call.getArgCount() > 1 ?
049 compiler.compileScalar(call.getArg(1), true) :
050 new ValueCalc(call);
051 return new AggregateCalc(call, listCalc, calc);
052 }
053
054 public static class AggregateCalc extends AbstractDoubleCalc {
055 private final ListCalc listCalc;
056 private final Calc calc;
057
058 public AggregateCalc(Exp exp, ListCalc listCalc, Calc calc) {
059 super(exp, new Calc[]{listCalc, calc});
060 this.listCalc = listCalc;
061 this.calc = calc;
062 }
063
064 public double evaluateDouble(Evaluator evaluator) {
065 Aggregator aggregator =
066 (Aggregator) evaluator.getProperty(
067 Property.AGGREGATION_TYPE.name, null);
068 if (aggregator == null) {
069 throw newEvalException(
070 null,
071 "Could not find an aggregator in the current evaluation context");
072 }
073 Aggregator rollup = aggregator.getRollup();
074 if (rollup == null) {
075 throw newEvalException(
076 null,
077 "Don't know how to rollup aggregator '" + aggregator + "'");
078 }
079 List list = evaluateCurrentList(listCalc, evaluator);
080 if (aggregator == RolapAggregator.DistinctCount) {
081 // If the list is empty, it means the current context
082 // contains no qualifying cells. The result set is empty.
083 if (list.size() == 0) {
084 return DoubleNull;
085 }
086
087 // Optimize the list
088 // E.g.
089 // List consists of:
090 // (Gender.[All Gender], [Product].[All Products]),
091 // (Gender.[All Gender].[F], [Product].[All Products].[Drink]),
092 // (Gender.[All Gender].[M], [Product].[All Products].[Food])
093 // Can be optimized to:
094 // (Gender.[All Gender], [Product].[All Products])
095 //
096 // Similar optimization can also be done for list of members.
097 List<Member[]> tupleList;
098 if (list.get(0) instanceof Member) {
099 tupleList = makeTupleList((List<Member>)list);
100 } else {
101 tupleList = (List<Member[]>) list;
102 }
103
104 RolapEvaluator rolapEvaluator = null;
105 if (evaluator instanceof RolapEvaluator) {
106 rolapEvaluator = (RolapEvaluator) evaluator;
107 }
108
109 if ((rolapEvaluator != null) &&
110 rolapEvaluator.getDialect().supportsUnlimitedValueList()) {
111 // If the DBMS does not have an upper limit on IN list
112 // predicate size, then don't attempt any list
113 // optimization, since the current algorithm is
114 // very slow. May want to revisit this if someone
115 // improves the algorithm.
116 } else {
117 // FIXME: We remove overlapping tuple entries only to pass
118 // AggregationOnDistinctCountMeasuresTest
119 // .testOptimizeListWithTuplesOfLength3 on Access. Without
120 // the optimization, we generate a statement 7000
121 // characters long and Access gives "Query is too complex".
122 // The optimization is expensive, so we only want to do it
123 // if the DBMS can't execute the query otherwise.
124 if ((rolapEvaluator != null) &&
125 rolapEvaluator.getDialect().isAccess() &&
126 false) {
127 tupleList = removeOverlappingTupleEntries(tupleList);
128 }
129 if (true) {
130 tupleList =
131 optimizeChildren(
132 tupleList,
133 evaluator.getSchemaReader(),
134 evaluator.getMeasureCube());
135 }
136 checkIfAggregationSizeIsTooLarge(tupleList);
137 }
138
139 // Can't aggregate distinct-count values in the same way
140 // which is used for other types of aggregations. To evaluate a
141 // distinct-count across multiple members, we need to gather
142 // the members together, then evaluate the collection of
143 // members all at once. To do this, we postpone evaluation,
144 // and create a lambda function containing the members.
145 Evaluator evaluator2 =
146 evaluator.pushAggregation(tupleList);
147 final Object o = evaluator2.evaluateCurrent();
148 final Number number = (Number) o;
149 return GenericCalc.numberToDouble(number);
150 }
151 return (Double) rollup.aggregate(evaluator.push(), list, calc);
152 }
153
154 /**
155 * In case of distinct count aggregation if a tuple which is a super
156 * set of other tuples in the set exists then the child tuples can be
157 * ignored.
158 *
159 * <p>
160 * E.g.
161 * List consists of:
162 * (Gender.[All Gender], [Product].[All Products]),
163 * (Gender.[All Gender].[F], [Product].[All Products].[Drink]),
164 * (Gender.[All Gender].[M], [Product].[All Products].[Food])
165 * Can be optimized to:
166 * (Gender.[All Gender], [Product].[All Products])
167 *
168 * @param list
169 */
170
171 public static List<Member[]> removeOverlappingTupleEntries(List<Member[]> list) {
172 List<Member[]> trimmedList = new ArrayList<Member[]>();
173 for (Member[] tuple1 : list) {
174 if (trimmedList.isEmpty()) {
175 trimmedList.add(tuple1);
176 } else {
177 boolean ignore = false;
178 final Iterator<Member[]> iterator = trimmedList.iterator();
179 while (iterator.hasNext()) {
180 Member[] tuple2 = iterator.next();
181 if (isSuperSet(tuple1, tuple2)) {
182 iterator.remove();
183 } else if (isSuperSet(tuple2, tuple1) ||
184 isEqual(tuple1, tuple2)) {
185 ignore = true;
186 break;
187 }
188 }
189 if (!ignore) {
190 trimmedList.add(tuple1);
191 }
192 }
193 }
194 return trimmedList;
195 }
196
197 /**
198 * Returns whether tuple1 is a superset of tuple2
199 * @param tuple1
200 * @param tuple2
201 * @return boolean
202 */
203 public static boolean isSuperSet(Member[] tuple1, Member[] tuple2) {
204 int parentLevelCount = 0;
205 for (int i = 0; i < tuple1.length; i++) {
206 Member member1 = tuple1[i];
207 Member member2 = tuple2[i];
208
209 if (!member2.isChildOrEqualTo(member1)) {
210 return false;
211 }
212 if (member1.getLevel().getDepth() < member2.getLevel().getDepth()) {
213 parentLevelCount++;
214 }
215 }
216 return parentLevelCount > 0;
217 }
218
219 /**
220 * Forms a list tuples from a list of members
221 * @param list of members
222 * @return list of tuples
223 */
224 public static List<Member[]> makeTupleList(List<Member> list) {
225 List<Member[]> tupleList = new ArrayList<Member[]>(list.size());
226 for (Member member : list) {
227 tupleList.add(new Member[] {member});
228 }
229 return tupleList;
230 }
231
232 private void checkIfAggregationSizeIsTooLarge(List list) {
233 final IntegerProperty property =
234 MondrianProperties.instance().MaxConstraints;
235 final int maxConstraints = property.get();
236 if (list.size() > maxConstraints) {
237 throw newEvalException(
238 null,
239 "Distinct Count aggregation is not supported over a list"
240 + " with more than " + maxConstraints + " predicates"
241 + " (see property " + property.getPath() + ")");
242 }
243 }
244
245 public Calc[] getCalcs() {
246 return new Calc[] {listCalc, calc};
247 }
248
249 public boolean dependsOn(Dimension dimension) {
250 if (dimension.isMeasures()) {
251 return true;
252 }
253 return anyDependsButFirst(getCalcs(), dimension);
254 }
255
256 /**
257 * In distinct Count aggregation, if tuple list is a result
258 * m.children * n.children then it can be optimized to m * n
259 *
260 * <p>
261 * E.g.
262 * List consist of:
263 * (Gender.[All Gender].[F], [Store].[All Stores].[USA]),
264 * (Gender.[All Gender].[F], [Store].[All Stores].[USA].[OR]),
265 * (Gender.[All Gender].[F], [Store].[All Stores].[USA].[CA]),
266 * (Gender.[All Gender].[F], [Store].[All Stores].[USA].[WA]),
267 * (Gender.[All Gender].[F], [Store].[All Stores].[CANADA])
268 * (Gender.[All Gender].[M], [Store].[All Stores].[USA]),
269 * (Gender.[All Gender].[M], [Store].[All Stores].[USA].[OR]),
270 * (Gender.[All Gender].[M], [Store].[All Stores].[USA].[CA]),
271 * (Gender.[All Gender].[M], [Store].[All Stores].[USA].[WA]),
272 * (Gender.[All Gender].[M], [Store].[All Stores].[CANADA])
273 * Can be optimized to:
274 * (Gender.[All Gender], [Store].[All Stores].[USA])
275 * (Gender.[All Gender], [Store].[All Stores].[CANADA])
276 *
277 * @param tuples Tuples
278 * @param reader Schema reader
279 * @param baseCubeForMeasure Cube
280 * @return xxxx
281 */
282 public static List<Member[]> optimizeChildren(
283 List<Member[]> tuples,
284 SchemaReader reader,
285 Cube baseCubeForMeasure)
286 {
287 Map<Member, Integer>[] membersOccurencesInTuple =
288 membersVersusOccurencesInTuple(tuples);
289 int tupleLength = tuples.get(0).length;
290
291 //noinspection unchecked
292 Set<Member>[] sets = new HashSet[tupleLength];
293 boolean optimized = false;
294 for (int i = 0; i < tupleLength; i++) {
295 if (areOccurencesEqual(membersOccurencesInTuple[i].values())) {
296 Set<Member> members = membersOccurencesInTuple[i].keySet();
297 int originalSize = members.size();
298 sets[i] =
299 optimizeMemberSet(
300 new HashSet<Member>(members),
301 reader,
302 baseCubeForMeasure);
303 if (sets[i].size() != originalSize) {
304 optimized = true;
305 }
306 }
307 }
308 if (optimized) {
309 if (sets.length == 1) {
310 Set<Member> set = sets[0];
311 List<Member[]> tupleList =
312 new ArrayList<Member[]>(set.size());
313 for (Member member : set) {
314 tupleList.add(new Member[] {member});
315 }
316 return tupleList;
317 }
318 return crossProd(sets);
319 }
320 return tuples;
321 }
322
323 /**
324 * Finds member occurrences in tuple and generates a map of Members
325 * versus their occurrences in tuples.
326 *
327 * @param tuples List of tuples
328 * @return Map of the number of occurrences of each member in a tuple
329 */
330 public static Map<Member, Integer>[] membersVersusOccurencesInTuple(
331 List<Member[]> tuples)
332 {
333
334 int tupleLength = tuples.get(0).length;
335 //noinspection unchecked
336 Map<Member, Integer>[] counters = new Map[tupleLength];
337 for (int i = 0; i < counters.length; i++) {
338 counters[i] = new HashMap<Member, Integer>();
339 }
340 for (Member[] tuple : tuples) {
341 for (int i = 0; i < tuple.length; i++) {
342 Member member = tuple[i];
343 Map<Member, Integer> map = counters[i];
344 if (map.containsKey(member)) {
345 Integer count = map.get(member);
346 map.put(member, ++count);
347 } else {
348 map.put(member, 1);
349 }
350 }
351 }
352 return counters;
353 }
354
355 private static Set<Member> optimizeMemberSet(
356 Set<Member> members,
357 SchemaReader reader,
358 Cube baseCubeForMeasure)
359 {
360
361 boolean didOptimize;
362 Set<Member> membersToBeOptimized = new HashSet<Member>();
363 Set<Member> optimizedMembers = new HashSet<Member>();
364 while (members.size() > 0) {
365 Iterator<Member> iterator = members.iterator();
366 Member first = iterator.next();
367 if (first.isAll()) {
368 optimizedMembers.clear();
369 optimizedMembers.add(first);
370 return optimizedMembers;
371 }
372 membersToBeOptimized.add(first);
373 iterator.remove();
374
375 Member firstParentMember = first.getParentMember();
376 while (iterator.hasNext()) {
377 Member current = iterator.next();
378 if (current.isAll()) {
379 optimizedMembers.clear();
380 optimizedMembers.add(current);
381 return optimizedMembers;
382 }
383
384 Member currentParentMember = current.getParentMember();
385 if (firstParentMember == null &&
386 currentParentMember == null ||
387 (firstParentMember != null &&
388 firstParentMember.equals(currentParentMember))) {
389 membersToBeOptimized.add(current);
390 iterator.remove();
391 }
392 }
393
394 int childCountOfParent = -1;
395 if (firstParentMember != null) {
396 childCountOfParent = getChildCount(firstParentMember, reader);
397 }
398 if (childCountOfParent != -1 &&
399 membersToBeOptimized.size() == childCountOfParent &&
400 canOptimize(firstParentMember,baseCubeForMeasure)) {
401 optimizedMembers.add(firstParentMember);
402 didOptimize = true;
403 } else {
404 optimizedMembers.addAll(membersToBeOptimized);
405 didOptimize = false;
406 }
407 membersToBeOptimized.clear();
408
409 if (members.size() == 0 && didOptimize) {
410 Set temp = members;
411 members = optimizedMembers;
412 optimizedMembers = temp;
413 }
414 }
415 return optimizedMembers;
416 }
417
418 /**
419 * Returns whether tuples are equal. They must have the same length.
420 *
421 * @param tuple1 First tuple
422 * @param tuple2 Second tuple
423 * @return whether tuples are equal
424 */
425 private static boolean isEqual(Member[] tuple1, Member[] tuple2) {
426 for (int i = 0; i < tuple1.length; i++) {
427 if (!tuple1[i].getUniqueName().
428 equals(tuple2[i].getUniqueName())) {
429 return false;
430 }
431 }
432 return true;
433 }
434
435 private static boolean canOptimize(
436 Member parentMember,
437 Cube baseCube)
438 {
439 return dimensionJoinsToBaseCube(
440 parentMember.getDimension(), baseCube)
441 || !parentMember.isAll();
442 }
443
444 private static List<Member[]> crossProd(Set<Member>[] sets) {
445 List<Member> firstList = new ArrayList<Member>(sets[0]);
446 List<Member> secondList = new ArrayList<Member>(sets[1]);
447 List<Member[]> tupleList =
448 CrossJoinFunDef.crossJoin(firstList, secondList);
449 for (int i = 2; i < sets.length; i++) {
450 Set<Member> set = sets[i];
451 tupleList =
452 CrossJoinFunDef.crossJoin(
453 tupleList, new ArrayList<Member>(set));
454 }
455 return tupleList;
456 }
457
458 private static boolean dimensionJoinsToBaseCube(
459 Dimension dimension,
460 Cube baseCube)
461 {
462 HashSet<Dimension> dimensions = new HashSet<Dimension>();
463 dimensions.add(dimension);
464 return baseCube.nonJoiningDimensions(dimensions).size() == 0;
465 }
466
467 private static int getChildCount(
468 Member parentMember,
469 SchemaReader reader)
470 {
471 int childrenCountFromCache =
472 reader.getChildrenCountFromCache(parentMember);
473 if (childrenCountFromCache != -1) {
474 return childrenCountFromCache;
475 }
476 return reader.getMemberChildren(parentMember).size();
477 }
478
479 }
480 }
481
482 // End AggregateFunDef.java