Merge pull request #624 from guiyj/develop

Address issue #622: merge FPC into FlowDroid

Author: StevenArzt

soot-infoflow-cmd/src/soot/jimple/infoflow/cmd/MainClass.java

@@ -108,6 +108,7 @@ public class MainClass {
 	private static final String OPTION_MAX_CALLBACKS_COMPONENT = "mc";
 	private static final String OPTION_MAX_CALLBACKS_DEPTH = "md";
 	private static final String OPTION_PATH_SPECIFIC_RESULTS = "ps";
+	private static final String OPTION_MAX_THREAD_NUMBER = "mt";
 
 	// Inter-component communication
 	private static final String OPTION_ICC_MODEL = "im";
@@ -126,6 +127,7 @@ public class MainClass {
 	private static final String OPTION_IMPLICIT_FLOW_MODE = "i";
 	private static final String OPTION_STATIC_FLOW_TRACKING_MODE = "sf";
 	private static final String OPTION_DATA_FLOW_DIRECTION = "dir";
+	private static final String OPTION_GC_SLEEP_TIME = "st";
 
 	// Evaluation-specific options
 	private static final String OPTION_ANALYZE_FRAMEWORKS = "ff";
@@ -193,7 +195,8 @@ private void initializeCommandLineOptions() {
 				"Compute the taint propagation paths and not just source-to-sink connections. This is a shorthand notation for -pr fast.");
 		options.addOption(OPTION_LOG_SOURCES_SINKS, "logsourcesandsinks", false,
 				"Write the discovered sources and sinks to the log output");
-		options.addOption("mt", "maxthreadnum", true, "Limit the maximum number of threads to the given value");
+		options.addOption(OPTION_MAX_THREAD_NUMBER, "maxthreadnum", true,
+				"Limit the maximum number of threads to the given value");
 		options.addOption(OPTION_ONE_COMPONENT, "onecomponentatatime", false,
 				"Analyze one Android component at a time");
 		options.addOption(OPTION_ONE_SOURCE, "onesourceatatime", false, "Analyze one source at a time");
@@ -241,6 +244,8 @@ private void initializeCommandLineOptions() {
 				"Use the specified mode when tracking static data flows (CONTEXTFLOWSENSITIVE, CONTEXTFLOWINSENSITIVE, NONE)");
 		options.addOption(OPTION_DATA_FLOW_DIRECTION, "direction", true,
 				"Specifies the direction of the infoflow analysis (FORWARDS, BACKWARDS)");
+		options.addOption(OPTION_GC_SLEEP_TIME, "gcsleeptime", true, 
+				"Specifies the sleep time for path edge collectors in seconds");
 
 		// Evaluation-specific options
 		options.addOption(OPTION_ANALYZE_FRAMEWORKS, "analyzeframeworks", false,
@@ -588,6 +593,8 @@ else if (solver.equalsIgnoreCase("FLOWINSENSITIVE"))
 			return DataFlowSolver.FlowInsensitive;
 		else if (solver.equalsIgnoreCase("GC"))
 			return DataFlowSolver.GarbageCollecting;
+		else if (solver.equalsIgnoreCase("FPC"))
+			return DataFlowSolver.FineGrainedGC;
 		else {
 			System.err.println(String.format("Invalid data flow solver: %s", solver));
 			throw new AbortAnalysisException();
@@ -790,6 +797,12 @@ private void parseCommandLineOptions(CommandLine cmd, InfoflowAndroidConfigurati
 			if (maxDepth != null)
 				config.getCallbackConfig().setMaxAnalysisCallbackDepth(maxDepth);
 		}
+		{
+			Integer maxthreadnum = getIntOption(cmd, OPTION_MAX_THREAD_NUMBER);
+			if (maxthreadnum != null) {
+				config.setMaxThreadNum(maxthreadnum);
+			}
+		}
 
 		// Inter-component communication
 		if (cmd.hasOption(OPTION_ICC_NO_PURIFY))
@@ -887,6 +900,13 @@ private void parseCommandLineOptions(CommandLine cmd, InfoflowAndroidConfigurati
 				config.getCallbackConfig().setCallbacksFile(callgraphFile);
 			}
 		}
+
+		{
+			Integer sleepTime = getIntOption(cmd, OPTION_GC_SLEEP_TIME);
+			if (sleepTime != null) {
+				config.getSolverConfiguration().setSleepTime(sleepTime);
+			}
+		}
 	}
 
 	private Integer getIntOption(CommandLine cmd, String option) {

soot-infoflow/src/soot/jimple/infoflow/AbstractInfoflow.java

@@ -18,6 +18,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import heros.solver.Pair;
 import soot.FastHierarchy;
 import soot.G;
 import soot.MethodOrMethodContext;
@@ -704,7 +705,13 @@ public Thread newThread(Runnable r) {
 			manager = initializeInfoflowManager(sourcesSinks, iCfg, globalTaintManager);
 
 			// Create the solver peer group
-			solverPeerGroup = new GCSolverPeerGroup();
+			switch (manager.getConfig().getSolverConfiguration().getDataFlowSolver()) {
+			case FineGrainedGC:
+				solverPeerGroup = new GCSolverPeerGroup<Pair<SootMethod, Abstraction>>();
+				break;
+			default:
+				solverPeerGroup = new GCSolverPeerGroup<SootMethod>();
+			}
 
 			// Initialize the alias analysis
 			Abstraction zeroValue = Abstraction.getZeroAbstraction(manager.getConfig().getFlowSensitiveAliasing());
@@ -1238,10 +1245,18 @@ protected IInfoflowSolver createDataFlowSolver(InterruptableExecutor executor, A
 			return new soot.jimple.infoflow.solver.fastSolver.flowInsensitive.InfoflowSolver(problem, executor);
 		case GarbageCollecting:
 			logger.info("Using garbage-collecting solver");
-			IInfoflowSolver solver = new soot.jimple.infoflow.solver.gcSolver.InfoflowSolver(problem, executor);
+			IInfoflowSolver solver = new soot.jimple.infoflow.solver.gcSolver.InfoflowSolver(problem, executor,
+					solverConfig.getSleepTime());
 			solverPeerGroup.addSolver(solver);
 			solver.setPeerGroup(solverPeerGroup);
 			return solver;
+		case FineGrainedGC:
+			logger.info("Using fine-grained garbage-collecting solver");
+			IInfoflowSolver fgSolver = new soot.jimple.infoflow.solver.gcSolver.fpc.InfoflowSolver(problem, executor,
+					solverConfig.getSleepTime());
+			solverPeerGroup.addSolver(fgSolver);
+			fgSolver.setPeerGroup(solverPeerGroup);
+			return fgSolver;
 		default:
 			throw new RuntimeException("Unsupported data flow solver");
 		}

soot-infoflow/src/soot/jimple/infoflow/InfoflowConfiguration.java

@@ -133,7 +133,12 @@ public static enum DataFlowSolver {
 		/**
 		 * Use the garbage-collecting solver
 		 */
-		GarbageCollecting
+		GarbageCollecting,
+
+		/**
+		 * Use the fine-grained GC solver
+		 * */
+		FineGrainedGC,
 	}
 
 	public static enum DataFlowDirection {
@@ -971,6 +976,7 @@ public static class SolverConfiguration {
 		private int maxJoinPointAbstractions = 10;
 		private int maxCalleesPerCallSite = 75;
 		private int maxAbstractionPathLength = 100;
+		private int sleepTime = 1;
 
 		/**
 		 * Copies the settings of the given configuration into this configuration object
@@ -1084,6 +1090,24 @@ public void setMaxAbstractionPathLength(int maxAbstractionPathLength) {
 			this.maxAbstractionPathLength = maxAbstractionPathLength;
 		}
 
+		/**
+		 * Sets the sleep time of garbage colletors
+		 * 
+		 * @param sleeptime The interval in second for the path edge collection
+		 */
+		public void setSleepTime(int sleeptime) {
+			this.sleepTime = sleeptime;
+		}
+
+		/**
+		 * Gets the sleep time of garbage colletors
+		 * 
+		 * @return The interval in second for the path edge collection
+		 */
+		public int getSleepTime() {
+			return this.sleepTime;
+		}
+
 		@Override
 		public int hashCode() {
 			final int prime = 31;

soot-infoflow/src/soot/jimple/infoflow/solver/gcSolver/AbstractGarbageCollector.java

@@ -11,23 +11,23 @@
  * @author Steven Arzt
  *
  */
-public abstract class AbstractGarbageCollector<N, D> implements IGarbageCollector<N, D> {
+public abstract class AbstractGarbageCollector<N, D, A> implements IGarbageCollector<N, D> {
 
 	protected final BiDiInterproceduralCFG<N, SootMethod> icfg;
-	protected final IGCReferenceProvider<D, N> referenceProvider;
-	protected final ConcurrentHashMultiMap<SootMethod, PathEdge<N, D>> jumpFunctions;
+	protected final IGCReferenceProvider<A> referenceProvider;
+	protected final ConcurrentHashMultiMap<A, PathEdge<N, D>> jumpFunctions;
 
 	public AbstractGarbageCollector(BiDiInterproceduralCFG<N, SootMethod> icfg,
-			ConcurrentHashMultiMap<SootMethod, PathEdge<N, D>> jumpFunctions,
-			IGCReferenceProvider<D, N> referenceProvider) {
+			ConcurrentHashMultiMap<A, PathEdge<N, D>> jumpFunctions,
+			IGCReferenceProvider<A> referenceProvider) {
 		this.icfg = icfg;
 		this.referenceProvider = referenceProvider;
 		this.jumpFunctions = jumpFunctions;
 		initialize();
 	}
 
 	public AbstractGarbageCollector(BiDiInterproceduralCFG<N, SootMethod> icfg,
-			ConcurrentHashMultiMap<SootMethod, PathEdge<N, D>> jumpFunctions) {
+			ConcurrentHashMultiMap<A, PathEdge<N, D>> jumpFunctions) {
 		this.icfg = icfg;
 		this.referenceProvider = createReferenceProvider();
 		this.jumpFunctions = jumpFunctions;
@@ -46,8 +46,10 @@ protected void initialize() {
 	 * 
 	 * @return The new reference provider
 	 */
-	protected IGCReferenceProvider<D, N> createReferenceProvider() {
-		return new OnDemandReferenceProvider<>(icfg);
+	protected abstract IGCReferenceProvider<A> createReferenceProvider();
+
+	protected long getRemainingPathEdgeCount() {
+		return jumpFunctions.values().size();
 	}
 
 }

soot-infoflow/src/soot/jimple/infoflow/solver/gcSolver/AbstractReferenceCountingGarbageCollector.java

@@ -18,17 +18,17 @@
  * @author Steven Arzt
  *
  */
-public abstract class AbstractReferenceCountingGarbageCollector<N, D> extends AbstractGarbageCollector<N, D>
-		implements IGarbageCollectorPeer {
-
-	private ConcurrentCountingMap<SootMethod> jumpFnCounter = new ConcurrentCountingMap<>();
-	private final Set<SootMethod> gcScheduleSet = new ConcurrentHashSet<>();
-	private final AtomicInteger gcedMethods = new AtomicInteger();
-	private final AtomicInteger gcedEdges = new AtomicInteger();
-	private final ExtendedAtomicInteger edgeCounterForThreshold = new ExtendedAtomicInteger();
-	private GarbageCollectionTrigger trigger = GarbageCollectionTrigger.Immediate;
-	private GarbageCollectorPeerGroup peerGroup = null;
-	private boolean checkChangeCounter = false;
+public abstract class AbstractReferenceCountingGarbageCollector<N, D, A> extends AbstractGarbageCollector<N, D, A>
+		implements IGarbageCollectorPeer<A> {
+
+	protected ConcurrentCountingMap<A> jumpFnCounter = new ConcurrentCountingMap<>();
+	protected final Set<A> gcScheduleSet = new ConcurrentHashSet<>();
+	protected final AtomicInteger gcedAbstractions = new AtomicInteger();
+	protected final AtomicInteger gcedEdges = new AtomicInteger();
+	protected final ExtendedAtomicInteger edgeCounterForThreshold = new ExtendedAtomicInteger();
+	protected GarbageCollectionTrigger trigger = GarbageCollectionTrigger.Immediate;
+	protected GarbageCollectorPeerGroup<A> peerGroup = null;
+	protected boolean checkChangeCounter = false;
 
 	protected boolean validateEdges = false;
 	protected Set<PathEdge<N, D>> oldEdges = new HashSet<>();
@@ -44,21 +44,23 @@ public abstract class AbstractReferenceCountingGarbageCollector<N, D> extends Ab
 	protected int edgeThreshold = 0;
 
 	public AbstractReferenceCountingGarbageCollector(BiDiInterproceduralCFG<N, SootMethod> icfg,
-			ConcurrentHashMultiMap<SootMethod, PathEdge<N, D>> jumpFunctions,
-			IGCReferenceProvider<D, N> referenceProvider) {
+			ConcurrentHashMultiMap<A, PathEdge<N, D>> jumpFunctions,
+			IGCReferenceProvider<A> referenceProvider) {
 		super(icfg, jumpFunctions, referenceProvider);
 	}
 
 	public AbstractReferenceCountingGarbageCollector(BiDiInterproceduralCFG<N, SootMethod> icfg,
-			ConcurrentHashMultiMap<SootMethod, PathEdge<N, D>> jumpFunctions) {
+			ConcurrentHashMultiMap<A, PathEdge<N, D>> jumpFunctions) {
 		super(icfg, jumpFunctions);
 	}
 
+	protected abstract A genAbstraction(PathEdge<N, D> edge);
+
 	@Override
 	public void notifyEdgeSchedule(PathEdge<N, D> edge) {
-		SootMethod sm = icfg.getMethodOf(edge.getTarget());
-		jumpFnCounter.increment(sm);
-		gcScheduleSet.add(sm);
+		A abstraction = genAbstraction(edge);
+		jumpFnCounter.increment(abstraction);
+		gcScheduleSet.add(abstraction);
 		if (trigger == GarbageCollectionTrigger.EdgeThreshold)
 			edgeCounterForThreshold.incrementAndGet();
 
@@ -70,42 +72,8 @@ public void notifyEdgeSchedule(PathEdge<N, D> edge) {
 
 	@Override
 	public void notifyTaskProcessed(PathEdge<N, D> edge) {
-		jumpFnCounter.decrement(icfg.getMethodOf(edge.getTarget()));
-	}
-
-	/**
-	 * Checks whether the given method has any open dependencies that prevent its
-	 * jump functions from being garbage collected
-	 * 
-	 * @param method           The method to check
-	 * @param referenceCounter The counter that keeps track of active references to
-	 *                         taint abstractions
-	 * @return True it the method has active dependencies and thus cannot be
-	 *         garbage-collected, false otherwise
-	 */
-	private boolean hasActiveDependencies(SootMethod method, ConcurrentCountingMap<SootMethod> referenceCounter) {
-		int changeCounter = -1;
-		do {
-			// Update the change counter for the next round
-			changeCounter = referenceCounter.getChangeCounter();
-
-			// Check the method itself
-			if (referenceCounter.get(method) > 0)
-				return true;
-
-			// Check the transitive callees
-			Set<SootMethod> references = referenceProvider.getMethodReferences(method, null);
-			for (SootMethod ref : references) {
-				if (referenceCounter.get(ref) > 0)
-					return true;
-			}
-		} while (checkChangeCounter && changeCounter != referenceCounter.getChangeCounter());
-		return false;
-	}
-
-	@Override
-	public boolean hasActiveDependencies(SootMethod method) {
-		return hasActiveDependencies(method, jumpFnCounter);
+		A abstraction = genAbstraction(edge);
+		jumpFnCounter.decrement(abstraction);
 	}
 
 	/**
@@ -120,18 +88,17 @@ protected void gcImmediate() {
 
 			// Perform the garbage collection if required
 			if (gc) {
-				int tempMethods = 0;
 				onBeforeRemoveEdges();
-				for (SootMethod sm : gcScheduleSet) {
+				for (A abst : gcScheduleSet) {
 					// Is it safe to remove this method?
 					if (peerGroup != null) {
-						if (peerGroup.hasActiveDependencies(sm))
+						if (peerGroup.hasActiveDependencies(abst))
 							continue;
-					} else if (hasActiveDependencies(sm))
+					} else if (hasActiveDependencies(abst))
 						continue;
 
 					// Get stats for the stuff we are about to remove
-					Set<PathEdge<N, D>> oldFunctions = jumpFunctions.get(sm);
+					Set<PathEdge<N, D>> oldFunctions = jumpFunctions.get(abst);
 					if (oldFunctions != null) {
 						int gcedSize = oldFunctions.size();
 						gcedEdges.addAndGet(gcedSize);
@@ -142,15 +109,14 @@ protected void gcImmediate() {
 					// First unregister the method, then delete the edges. In case some other thread
 					// concurrently schedules a new edge, the method gets back into the GC work list
 					// this way.
-					gcScheduleSet.remove(sm);
-					if (jumpFunctions.remove(sm)) {
-						gcedMethods.incrementAndGet();
-						tempMethods++;
+					gcScheduleSet.remove(abst);
+					if (jumpFunctions.remove(abst)) {
+						gcedAbstractions.incrementAndGet();
 						if (validateEdges)
 							oldEdges.addAll(oldFunctions);
 					}
 				}
-				onAfterRemoveEdges(tempMethods);
+				onAfterRemoveEdges();
 			}
 		}
 	}
@@ -168,12 +134,12 @@ protected void onBeforeRemoveEdges() {
 	 * 
 	 * @param gcedMethods The number of methods for which edges have been removed
 	 */
-	protected void onAfterRemoveEdges(int gcedMethods) {
+	protected void onAfterRemoveEdges() {
 	}
 
 	@Override
-	public int getGcedMethods() {
-		return gcedMethods.get();
+	public int getGcedAbstractions() {
+		return gcedAbstractions.get();
 	}
 
 	@Override
@@ -219,7 +185,7 @@ public void setTrigger(GarbageCollectionTrigger trigger) {
 	 * 
 	 * @param peerGroup The peer group
 	 */
-	public void setPeerGroup(GarbageCollectorPeerGroup peerGroup) {
+	public void setPeerGroup(GarbageCollectorPeerGroup<A> peerGroup) {
 		this.peerGroup = peerGroup;
 		peerGroup.addGarbageCollector(this);
 	}

soot-infoflow/src/soot/jimple/infoflow/solver/gcSolver/AbstractReferenceProvider.java

@@ -14,7 +14,7 @@
  * @author Steven Arzt
  *
  */
-public abstract class AbstractReferenceProvider<D, N> implements IGCReferenceProvider<D, N> {
+public abstract class AbstractReferenceProvider<A, N> implements IGCReferenceProvider<A> {
 
 	protected final BiDiInterproceduralCFG<N, SootMethod> icfg;