本文整理汇总了Java中org.apache.pig.newplan.logical.relational.LogicalPlan类的典型用法代码示例。如果您正苦于以下问题:Java LogicalPlan类的具体用法?Java LogicalPlan怎么用?Java LogicalPlan使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
LogicalPlan类属于org.apache.pig.newplan.logical.relational包,在下文中一共展示了LogicalPlan类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: testSimpleNotPossible
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testSimpleNotPossible() throws Exception {
String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
"B = FOREACH A GENERATE name, flatten(cuisines) as cuisines;" +
"C = FILTER B BY cuisines == 'pizza';" +
"D = STORE C INTO 'empty';" ;
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( fe1 instanceof LOForEach );
Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
Assert.assertTrue( fe2 instanceof LOForEach );
Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:18,代码来源:TestNewPlanFilterAboveForeach.java
示例2: testSimple
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testSimple() throws Exception {
String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
"B = FOREACH A GENERATE name, flatten(cuisines);" +
"C = FILTER B BY name == 'joe';" +
"D = STORE C INTO 'empty';" ;
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
Operator fe1 = newLogicalPlan.getSuccessors( filter ).get( 0 );
Assert.assertTrue( fe1 instanceof LOForEach );
Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
Assert.assertTrue( fe2 instanceof LOForEach );
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:18,代码来源:TestNewPlanFilterAboveForeach.java
示例3: testFilterFilter
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterFilter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = filter A by $0 != 'name';" +
"C = filter B by $1 < 18;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
Assert.assertTrue( ((LOFilter)op).getAlias().equals( "B" ) );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
Assert.assertTrue( ((LOFilter)op).getAlias().equals( "C" ) );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:23,代码来源:TestNewPlanPushUpFilter.java
示例4: visit
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void visit(LOSplit split) throws FrontendException {
List<Operator> branchOutputs = split.getPlan().getSuccessors(split);
for (int i=0;i<branchOutputs.size();i++) {
Operator branchOutput = branchOutputs.get(i);
Set<Long> branchOutputUids = (Set<Long>)branchOutput.getAnnotation(ColumnPruneHelper.INPUTUIDS);
if (branchOutputUids!=null) {
Set<Integer> columnsToDrop = new HashSet<Integer>();
for (int j=0;j<split.getSchema().size();j++) {
if (!branchOutputUids.contains(split.getSchema().getField(j).uid))
columnsToDrop.add(j);
}
if (!columnsToDrop.isEmpty()) {
LOForEach foreach = Util.addForEachAfter((LogicalPlan)split.getPlan(), split, i, columnsToDrop);
foreach.getSchema();
}
}
}
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:24,代码来源:ColumnPruneVisitor.java
示例5: removeSubTree
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
private void removeSubTree(LogicalRelationalOperator op) throws FrontendException {
LogicalPlan p = (LogicalPlan)op.getPlan();
List<Operator> ll = p.getPredecessors(op);
if (ll != null) {
for(Operator pred: ll.toArray(new Operator[ll.size()])) {
removeSubTree((LogicalRelationalOperator)pred);
}
}
if (p.getSuccessors(op) != null) {
Operator[] succs = p.getSuccessors(op).toArray(new Operator[0]);
for(Operator s: succs) {
p.disconnect(op, s);
}
}
p.remove(op);
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:19,代码来源:ColumnPruneVisitor.java
示例6: test3
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
* No fields are used in filter condition at all.
* @throws Exception
*/
@Test
public void test3() throws Exception {
String query = "A =LOAD 'file.txt' AS (a:(u,v), b, c);" +
"B = FOREACH A GENERATE $0, b;" +
"C = FILTER B BY 8 > 5;" +
"STORE C INTO 'empty';";
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
Operator fe1 = newLogicalPlan.getSuccessors( filter ).get( 0 );
Assert.assertTrue( fe1 instanceof LOForEach );
Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
Assert.assertTrue( fe2 instanceof LOForEach );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:22,代码来源:TestNewPlanFilterAboveForeach.java
示例7: checkWarning
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
private void checkWarning(String query, String warnMsg) throws FrontendException {
LogicalPlan plan = generateLogicalPlan(query);
new ColumnAliasConversionVisitor( plan ).visit();
// validate
CompilationMessageCollector collector = new CompilationMessageCollector();
TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
typeChecker.visit();
new CastLineageSetter(plan, collector).visit();
printMessageCollector(collector);
boolean isWarningSeen = false;
assertTrue("message collector has message", collector.hasMessage());
for (Message msg : collector){
if (msg.getMessageType() == MessageType.Warning
&& msg.getMessage().contains(warnMsg)){
isWarningSeen = true;
}
}
assertTrue("Expected warning is seen", isWarningSeen);
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:24,代码来源:TestTypeCheckingValidatorNewLP.java
示例8: testFilterAfterNestedLimit
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
* Test that filter cannot get pushed up over nested Limit (see PIG-3347)
*/
@Test
public void testFilterAfterNestedLimit() throws Exception {
String query = "a = LOAD 'file.txt';" +
"a_group = group a by $0;" +
"b = foreach a_group { a_limit = limit a.$0 5;generate group, a_limit;}" +
"c = filter b by SIZE(a_limit) == 1;" +
"store c into 'empty';";
// filter should not be pushed above nested distinct,
//ie expect - loload -> locogroup -> foreach -> filter
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
newLogicalPlan.explain(System.out, "text", true);
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator cogroup = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( cogroup instanceof LOCogroup );
Operator foreach = newLogicalPlan.getSuccessors(cogroup).get( 0 );
Assert.assertTrue( foreach instanceof LOForEach );
Operator filter = newLogicalPlan.getSuccessors(foreach).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:26,代码来源:TestNewPlanFilterRule.java
示例9: testBuildLoadOpWithDefaultFunc
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testBuildLoadOpWithDefaultFunc() throws Exception {
String query = "a = load '1.txt';" +
"store a into 'output';";
LogicalPlan lp = buildPlan(query);
FuncSpec funcSpec = getFirstLoadFuncSpec(lp);
assertEquals("org.apache.pig.builtin.PigStorage", funcSpec.getClassName());
// set default load func in config
pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_DEFAULT_LOAD_FUNC, "org.apache.pig.test.PigStorageWithSchema");
query = "a = load '1.txt';" +
"store a into 'output';";
lp = buildPlan(query);
funcSpec = getFirstLoadFuncSpec(lp);
assertEquals("org.apache.pig.test.PigStorageWithSchema", funcSpec.getClassName());
// unset default load func
pigServer.getPigContext().getProperties().remove(PigConfiguration.PIG_DEFAULT_LOAD_FUNC);
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:20,代码来源:TestLogicalPlanBuilder.java
示例10: testMultiQueryWithTwoStores
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testMultiQueryWithTwoStores() {
System.out.println("===== multi-query with 2 stores =====");
try {
myPig.setBatchOn();
myPig.registerQuery("a = load 'passwd' " +
"using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);");
myPig.registerQuery("b = filter a by uid > 5;");
myPig.registerQuery("store b into '/tmp/output1';");
myPig.registerQuery("c = group b by gid;");
myPig.registerQuery("store c into '/tmp/output2';");
LogicalPlan lp = checkLogicalPlan(1, 2, 5);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 11);
checkMRPlan(pp, 1, 1, 1);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
}
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:27,代码来源:TestMultiQueryCompiler.java
示例11: testFilterConstantConditionGroupBy
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterConstantConditionGroupBy() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:21,代码来源:TestNewPlanPushUpFilter.java
示例12: testNotPossibleFilter
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testNotPossibleFilter() throws Exception {
String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
"B = FOREACH A GENERATE name, flatten(cuisines);" +
"C = FILTER B BY cuisine == 'French';" +
"D = STORE C INTO 'empty';";
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( fe1 instanceof LOForEach );
Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
Assert.assertTrue( fe2 instanceof LOForEach );
Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:18,代码来源:TestNewPlanFilterAboveForeach.java
示例13: testFilterConstantConditionGroupBy
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterConstantConditionGroupBy() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:19,代码来源:TestNewPlanPushUpFilter.java
示例14: createDummyRelOpWithAlias
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
* @return a dummy logical relational operator
*/
private LogicalRelationalOperator createDummyRelOpWithAlias() {
class DummyRelOp extends LogicalRelationalOperator{
DummyRelOp(){
super("dummy", new LogicalPlan());
this.alias = "dummy";
}
@Override
public LogicalSchema getSchema() throws FrontendException {
return null;
}
@Override
public void accept(PlanVisitor v) throws FrontendException {
}
@Override
public boolean isEqual(Operator operator) throws FrontendException {
return false;
}
}
return new DummyRelOp();
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:29,代码来源:TestTypeCheckingValidatorNewLP.java
示例15: attachStorePlan
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
Operator input, String alias, PigContext pigContext) throws FrontendException {
func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;
FuncSpec funcSpec = new FuncSpec( func );
StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
fileName = removeQuotes( fileName );
FileSpec fileSpec = new FileSpec( fileName, funcSpec );
String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
stoFunc.setStoreFuncUDFContextSignature(sig);
LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
store.setAlias(alias);
try {
stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
} catch (IOException ioe) {
FrontendException e = new FrontendException( ioe.getMessage(), ioe );
throw e;
}
lp.add( store );
lp.connect( input, store );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:25,代码来源:QueryParserUtils.java
示例16: testFilterUDFGroupByOuter
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterUDFGroupByOuter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0 outer;" +
"C = filter B by " + MyFilterFunc.class.getName() + "($1) ;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:21,代码来源:TestNewPlanPushUpFilter.java
示例17: testFilterUDFFRJoin1
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
// UDF takes all input, so filter connot be pushed up.
public void testFilterUDFFRJoin1() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0 using 'replicated';" +
"D = filter C by TupleSize(*) > 5;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = newLogicalPlan.getSuccessors( loads.get( 0 ) ).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors( loads.get( 1 ) ).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:24,代码来源:TestNewPlanPushUpFilter.java
示例18: testFilterConstantConditionSort
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterConstantConditionSort() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = order A by $1, $2;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOSort );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:21,代码来源:TestNewPlanPushUpFilter.java
示例19: testFilterUDFForeach
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterUDFForeach() throws Exception {
String query = "A =load 'myfile' as (name, age, gpa);" +
"B = foreach A generate $1, $2;" +
"C = filter B by " + MyFilterFunc.class.getName() + "($1) ;" +
"D = STORE C INTO 'empty';";
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
Operator fe = newLogicalPlan.getSuccessors( filter ).get( 0 );
Assert.assertTrue( fe instanceof LOForEach );
Operator store = newLogicalPlan.getSuccessors( fe ).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:19,代码来源:TestNewPlanFilterAboveForeach.java
示例20: testForeachLimit
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testForeachLimit() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = foreach A generate $0, $1, flatten($2);" +
"C = limit B 10;" +
"store C into 'output';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator foreach = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( foreach instanceof LOForEach );
foreach = newLogicalPlan.getSuccessors( foreach ).get( 0 );
Assert.assertTrue( foreach instanceof LOForEach );
Assert.assertTrue( OptimizerUtils.hasFlatten( (LOForEach)foreach ) );
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:18,代码来源:TestNewPlanPushDownForeachFlatten.java
注:本文中的org.apache.pig.newplan.logical.relational.LogicalPlan类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论