api, db4s, common, webui: Allow specific named users to exceed database upload limits

Author: justinclift

api/handlers.go

@@ -665,16 +665,24 @@ func tagsHandler(w http.ResponseWriter, r *http.Request) {
 //      should be appended to.  For new databases it's not needed, but for existing databases it's required (its used to
 //      detect out of date / conflicting uploads)
 func uploadHandler(w http.ResponseWriter, r *http.Request) {
-	// Set the maximum accepted database size for uploading
-	r.Body = http.MaxBytesReader(w, r.Body, com.MaxDatabaseSize*1024*1024)
-
 	// Authenticate the request
 	loggedInUser, err := checkAuth(w, r)
 	if err != nil {
 		jsonErr(w, err.Error(), http.StatusUnauthorized)
 		return
 	}
 
+	// Set the maximum accepted database size for uploading
+	oversizeAllowed := false
+	for _, user := range com.Conf.Environment.SizeOverrideUsers {
+		if loggedInUser == user {
+			oversizeAllowed = true
+		}
+	}
+	if !oversizeAllowed {
+		r.Body = http.MaxBytesReader(w, r.Body, com.MaxDatabaseSize*1024*1024)
+	}
+
 	// Extract the database name and (optional) commit ID for the database from the request
 	_, dbName, commitID, err := com.GetFormODC(r)
 	if err != nil {
@@ -691,14 +699,15 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Check whether the uploaded database is too large
-	// TODO: Have a list of users (from the config.toml file) which don't have this check applied
-	if r.ContentLength > (com.MaxDatabaseSize * 1024 * 1024) {
-		jsonErr(w,
-			fmt.Sprintf("Database is too large. Maximum database upload size is %d MB, yours is %d MB",
-				com.MaxDatabaseSize, r.ContentLength/1024/1024), http.StatusBadRequest)
-		log.Println(fmt.Sprintf("'%s' attempted to upload an oversized database %d MB in size.  Limit is %d MB\n",
-			loggedInUser, r.ContentLength/1024/1024, com.MaxDatabaseSize))
-		return
+	if !oversizeAllowed {
+		if r.ContentLength > (com.MaxDatabaseSize * 1024 * 1024) {
+			jsonErr(w,
+				fmt.Sprintf("Database is too large. Maximum database upload size is %d MB, yours is %d MB",
+					com.MaxDatabaseSize, r.ContentLength/1024/1024), http.StatusBadRequest)
+			log.Println(fmt.Sprintf("'%s' attempted to upload an oversized database %d MB in size.  Limit is %d MB\n",
+				loggedInUser, r.ContentLength/1024/1024, com.MaxDatabaseSize))
+			return
+		}
 	}
 
 	// Process the upload

common/sqlite.go

@@ -865,7 +865,7 @@ func SQLiteRunQueryDefensive(w http.ResponseWriter, r *http.Request, querySource
 	var memUsed, memHighWater int64
 	memUsed, memHighWater, dataRows, err = SQLiteRunQuery(sdb, querySource, query, false, false)
 	if err != nil {
-		log.Printf("Error when preparing statement by '%s' for database (%s%s%s): '%s'\n",  SanitiseLogString(loggedInUser),
+		log.Printf("Error when preparing statement by '%s' for database (%s%s%s): '%s'\n", SanitiseLogString(loggedInUser),
 			SanitiseLogString(dbOwner), SanitiseLogString(dbFolder), SanitiseLogString(dbName), SanitiseLogString(err.Error()))
 		return SQLiteRecordSet{}, err
 	}

common/types.go

@@ -133,8 +133,9 @@ type DiskCacheInfo struct {
 // EnvInfo holds information about the purpose of the running server.  eg "is this a production, docker,
 // or development" instance?
 type EnvInfo struct {
-	Environment  string
-	UserOverride string `toml:"user_override"`
+	Environment       string
+	UserOverride      string   `toml:"user_override"`
+	SizeOverrideUsers []string `toml:"size_override_users"` // List of users allowed to override the database upload size limits
 }
 
 // EventProcessingInfo hold configuration for the event processing loop

db4s/main.go

@@ -784,7 +784,15 @@ func metadataGetHandler(w http.ResponseWriter, r *http.Request) {
 //
 func postHandler(w http.ResponseWriter, r *http.Request, userAcc string) {
 	// Set the maximum accepted database size for uploading
-	r.Body = http.MaxBytesReader(w, r.Body, com.MaxDatabaseSize*1024*1024)
+	oversizeAllowed := false
+	for _, user := range com.Conf.Environment.SizeOverrideUsers {
+		if userAcc == user {
+			oversizeAllowed = true
+		}
+	}
+	if !oversizeAllowed {
+		r.Body = http.MaxBytesReader(w, r.Body, com.MaxDatabaseSize*1024*1024)
+	}
 
 	// The "public" user isn't allowed to make changes
 	if userAcc == "public" {
@@ -812,14 +820,15 @@ func postHandler(w http.ResponseWriter, r *http.Request, userAcc string) {
 	}
 
 	// Check whether the uploaded database is too large
-	// TODO: Have a list of users (from the config.toml file) which don't have this check applied
-	if r.ContentLength > (com.MaxDatabaseSize * 1024 * 1024) {
-		http.Error(w,
-			fmt.Sprintf("Database is too large. Maximum database upload size is %d MB, yours is %d MB",
-				com.MaxDatabaseSize, r.ContentLength/1024/1024), http.StatusBadRequest)
-		log.Println(fmt.Sprintf("'%s' attempted to upload an oversized database %d MB in size.  Limit is %d MB\n",
-			userAcc, r.ContentLength/1024/1024, com.MaxDatabaseSize))
-		return
+	if !oversizeAllowed {
+		if r.ContentLength > (com.MaxDatabaseSize * 1024 * 1024) {
+			http.Error(w,
+				fmt.Sprintf("Database is too large. Maximum database upload size is %d MB, yours is %d MB",
+					com.MaxDatabaseSize, r.ContentLength/1024/1024), http.StatusBadRequest)
+			log.Println(fmt.Sprintf("'%s' attempted to upload an oversized database %d MB in size.  Limit is %d MB\n",
+				userAcc, r.ContentLength/1024/1024, com.MaxDatabaseSize))
+			return
+		}
 	}
 
 	// Do the remaining input validation, and add the database to the system in the appropriate spot

docker/config.toml

@@ -11,6 +11,7 @@ directory = "/home/dbhub/.dbhub/disk_cache"
 [environment]
 environment = "test"
 user_override = "default"
+size_override_users = ["default"]
 
 [event]
 delay = 2

webui/main.go

@@ -5165,9 +5165,6 @@ func uploadDataHandler(w http.ResponseWriter, r *http.Request) {
 	// TODO: Investigate getting the last modified timestamp of the database file selected for upload
 	// TODO   * https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified
 
-	// Set the maximum accepted database size for uploading
-	r.Body = http.MaxBytesReader(w, r.Body, com.MaxDatabaseSize*1024*1024)
-
 	// Retrieve session data (if any)
 	loggedInUser, validSession, err := checkLogin(r)
 	if err != nil {
@@ -5181,14 +5178,25 @@ func uploadDataHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Check whether the uploaded database is too large
-	if r.ContentLength > (com.MaxDatabaseSize * 1024 * 1024) {
-		errorPage(w, r, http.StatusBadRequest,
-			fmt.Sprintf("Database is too large. Maximum database upload size is %d MB, yours is %d MB",
-				com.MaxDatabaseSize, r.ContentLength/1024/1024))
-		log.Println(fmt.Sprintf("'%s' attempted to upload an oversized database %d MB in size.  Limit is %d MB\n",
-			loggedInUser, r.ContentLength/1024/1024, com.MaxDatabaseSize))
-		return
+	// Set the maximum accepted database size for uploading
+	oversizeAllowed := false
+	for _, user := range com.Conf.Environment.SizeOverrideUsers {
+		if loggedInUser == user {
+			oversizeAllowed = true
+		}
+	}
+	if !oversizeAllowed {
+		r.Body = http.MaxBytesReader(w, r.Body, com.MaxDatabaseSize*1024*1024)
+
+		// Check whether the uploaded database is too large (except for specific users)
+		if r.ContentLength > (com.MaxDatabaseSize * 1024 * 1024) {
+			errorPage(w, r, http.StatusBadRequest,
+				fmt.Sprintf("Database is too large. Maximum database upload size is %d MB, yours is %d MB",
+					com.MaxDatabaseSize, r.ContentLength/1024/1024))
+			log.Println(fmt.Sprintf("'%s' attempted to upload an oversized database %d MB in size.  Limit is %d MB\n",
+				loggedInUser, r.ContentLength/1024/1024, com.MaxDatabaseSize))
+			return
+		}
 	}
 
 	// Prepare the form data