Migrate away from the broken macvendors.co

mobile/src/freedom/java/be/mygod/vpnhotspot/util/UpdateChecker.kt

@@ -81,16 +81,17 @@ object UpdateChecker {
             val lastFetched = app.pref.getLong(KEY_LAST_FETCHED, -1)
             if (lastFetched in 0..now) delay(lastFetched + UPDATE_INTERVAL - now)
             currentCoroutineContext().ensureActive()
-            val conn = URL("https://api.github.com/repos/Mygod/VPNHotspot/releases?per_page=100")
-                .openConnection() as HttpURLConnection
             var reset: Long? = null
             app.pref.edit {
                 try {
-                    conn.setRequestProperty("Accept", "application/vnd.github.v3+json")
+                    val update = connectCancellable(
-                    val update = findUpdate(JSONArray(withContext(Dispatchers.IO) {
+                        "https://api.github.com/repos/Mygod/VPNHotspot/releases?per_page=100") { conn ->
-                        reset = conn.getHeaderField("X-RateLimit-Reset")?.toLongOrNull()
+                        conn.setRequestProperty("Accept", "application/vnd.github.v3+json")
-                        conn.inputStream.bufferedReader().readText()
+                        findUpdate(JSONArray(withContext(Dispatchers.IO) {
-                    }))
+                            reset = conn.getHeaderField("X-RateLimit-Reset")?.toLongOrNull()
+                            conn.inputStream.bufferedReader().readText()
+                        }))
+                    }
                     putString(KEY_VERSION, update?.let {
                         putLong(KEY_PUBLISHED, update.published)
                         it.message
@@ -103,7 +104,6 @@ object UpdateChecker {
                 } catch (e: Exception) {
                     Timber.w(e)
                 } finally {
-                    conn.disconnect()
                     putLong(KEY_LAST_FETCHED, System.currentTimeMillis())
                 }
             }

mobile/src/main/java/be/mygod/vpnhotspot/client/MacLookup.kt

@@ -6,16 +6,23 @@ import androidx.annotation.MainThread
 import be.mygod.vpnhotspot.App.Companion.app
 import be.mygod.vpnhotspot.R
 import be.mygod.vpnhotspot.room.AppDatabase
+import be.mygod.vpnhotspot.util.connectCancellable
 import be.mygod.vpnhotspot.widget.SmartSnackbar
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
 import org.json.JSONException
 import org.json.JSONObject
 import timber.log.Timber
-import java.net.HttpURLConnection
+import java.io.File
-import java.net.URL
+import java.io.FileNotFoundException
+import java.io.IOException
+import java.net.HttpCookie
+import java.util.Scanner
+import java.util.regex.Pattern
 
 /**
  * This class generates a default nickname for new clients.
@@ -29,50 +36,97 @@ object MacLookup {
         override fun getLocalizedMessage() = formatMessage(app)
     }
 
-    private val macLookupBusy = mutableMapOf<MacAddress, Pair<HttpURLConnection, Job>>()
+    private object SessionManager {
+        private const val CACHE_FILENAME = "maclookup_sessioncache"
+        private const val COOKIE_SESSION = "mac_address_vendor_lookup_session"
+        private val csrfPattern = Pattern.compile("<meta\\s+name=\"csrf-token\"\\s+content=\"([^\"]*)\"",
+            Pattern.CASE_INSENSITIVE)
+        private var sessionCache: List<String>?
+            get() = try {
+                File(app.deviceStorage.cacheDir, CACHE_FILENAME).readText().split('\n', limit = 2)
+            } catch (_: FileNotFoundException) {
+                null
+            }
+            set(value) = File(app.deviceStorage.cacheDir, CACHE_FILENAME).run {
+                if (value != null) writeText(value.joinToString("\n")) else if (!delete()) writeText("")
+            }
+        private val mutex = Mutex()
+
+        private suspend fun refreshSessionCache() = connectCancellable("https://macaddress.io/api") { conn ->
+            val cookies = conn.headerFields["set-cookie"] ?: throw IOException("Missing cookies")
+            var mavls: HttpCookie? = null
+            for (header in cookies) for (cookie in HttpCookie.parse(header)) {
+                if (cookie.name == COOKIE_SESSION) mavls = cookie
+            }
+            if (mavls == null) throw IOException("Missing set-cookie $COOKIE_SESSION")
+            val token = conn.inputStream.use { Scanner(it).findWithinHorizon(csrfPattern, 0) }
+                ?: throw IOException("Missing csrf-token")
+            listOf(mavls.toString(), csrfPattern.matcher(token).run {
+                check(matches())
+                group(1)!!
+            }).also { sessionCache = it }
+        }
+
+        suspend fun obtain(forceNew: Boolean): Pair<HttpCookie, String> = mutex.withLock {
+            val sessionCache = (if (forceNew) null else sessionCache) ?: refreshSessionCache()
+            HttpCookie.parse(sessionCache[0]).single() to sessionCache[1]
+        }
+    }
+
+    private val macLookupBusy = mutableMapOf<MacAddress, Job>()
     // http://en.wikipedia.org/wiki/ISO_3166-1
     private val countryCodeRegex = "(?:^|[^A-Z])([A-Z]{2})[\\s\\d]*$".toRegex()
 
     @MainThread
-    fun abort(mac: MacAddress) = macLookupBusy.remove(mac)?.let { (conn, job) ->
+    fun abort(mac: MacAddress) = macLookupBusy.remove(mac)?.cancel()
-        job.cancel()
-        conn.disconnect()
-    }
 
     @MainThread
     fun perform(mac: MacAddress, explicit: Boolean = false) {
         abort(mac)
-        val conn = URL("https://macvendors.co/api/$mac").openConnection() as HttpURLConnection
+        macLookupBusy[mac] = GlobalScope.launch(Dispatchers.IO) {
-        macLookupBusy[mac] = conn to GlobalScope.launch(Dispatchers.IO) {
             try {
-                val response = conn.inputStream.bufferedReader().readText()
+                var response: String? = null
-                val obj = JSONObject(response).getJSONObject("result")
+                for (tries in 0 until 5) {
-                obj.opt("error")?.also { throw UnexpectedError(mac, it.toString()) }
+                    val (cookie, csrf) = SessionManager.obtain(tries > 0)
-                val company = obj.getString("company")
+                    response = connectCancellable("https://macaddress.io/mac-address-lookup") { conn ->
-                val match = extractCountry(mac, response, obj)
+                        conn.requestMethod = "POST"
-                val result = if (match != null) {
+                        conn.setRequestProperty("content-type", "application/json")
-                    String(match.groupValues[1].flatMap { listOf('\uD83C', it + 0xDDA5) }.toCharArray()) + ' ' + company
+                        conn.setRequestProperty("cookie", "${cookie.name}=${cookie.value}")
-                } else company
+                        conn.setRequestProperty("x-csrf-token", csrf)
+                        conn.outputStream.writer().use { it.write("{\"macAddress\":\"$mac\",\"not-web-search\":true}") }
+                        when (val responseCode = conn.responseCode) {
+                            200 -> conn.inputStream.bufferedReader().readText()
+                            419 -> null
+                            else -> throw IOException("Unhandled response code $responseCode")
+                        }
+                    }
+                    if (response != null) break
+                }
+                if (response == null) throw IOException("Session creation failure")
+                val obj = JSONObject(response)
+                val result = if (obj.getJSONObject("blockDetails").getBoolean("blockFound")) {
+                    val vendor = obj.getJSONObject("vendorDetails")
+                    val company = vendor.getString("companyName")
+                    val match = extractCountry(mac, response, vendor)
+                    if (match != null) {
+                        String(match.groupValues[1].flatMap { listOf('\uD83C', it + 0xDDA5) }.toCharArray()) + ' ' +
+                                company
+                    } else company
+                } else null
                 AppDatabase.instance.clientRecordDao.upsert(mac) {
-                    nickname = result
+                    if (result != null) nickname = result
                     macLookupPending = false
                 }
-            } catch (e: JSONException) {
-                if ((e as? UnexpectedError)?.error == "no result") {
-                    // no vendor found, we should not retry in the future
-                    AppDatabase.instance.clientRecordDao.upsert(mac) { macLookupPending = false }
-                } else Timber.w(e)
-                if (explicit) SmartSnackbar.make(e).show()
             } catch (e: Throwable) {
-                Timber.d(e)
+                Timber.w(e)
                 if (explicit) SmartSnackbar.make(e).show()
             }
         }
     }
 
     private fun extractCountry(mac: MacAddress, response: String, obj: JSONObject): MatchResult? {
-        countryCodeRegex.matchEntire(obj.optString("country"))?.also { return it }
+        countryCodeRegex.matchEntire(obj.optString("countryCode"))?.also { return it }
-        val address = obj.optString("address")
+        val address = obj.optString("companyAddress")
         if (address.isBlank()) return null
         countryCodeRegex.find(address)?.also { return it }
         Timber.w(UnexpectedError(mac, response))

mobile/src/main/java/be/mygod/vpnhotspot/net/wifi/WifiSsidCompat.kt

@@ -63,7 +63,5 @@ data class WifiSsidCompat(val bytes: ByteArray) : Parcelable {
         return true
     }
 
-    override fun hashCode(): Int {
+    override fun hashCode() = bytes.contentHashCode()
-        return bytes.contentHashCode()
-    }
 }

mobile/src/main/java/be/mygod/vpnhotspot/util/Utils.kt

@@ -19,15 +19,20 @@ import androidx.fragment.app.FragmentManager
 import be.mygod.vpnhotspot.App.Companion.app
 import be.mygod.vpnhotspot.net.MacAddressCompat
 import be.mygod.vpnhotspot.widget.SmartSnackbar
+import kotlinx.coroutines.InternalCoroutinesApi
+import kotlinx.coroutines.job
 import timber.log.Timber
 import java.lang.invoke.MethodHandles
 import java.lang.reflect.InvocationHandler
 import java.lang.reflect.InvocationTargetException
 import java.lang.reflect.Method
+import java.net.HttpURLConnection
 import java.net.InetAddress
 import java.net.NetworkInterface
 import java.net.SocketException
-import java.util.*
+import java.net.URL
+import java.util.Locale
+import kotlin.coroutines.coroutineContext
 
 tailrec fun Throwable.getRootCause(): Throwable {
     if (this is InvocationTargetException || this is RemoteException) return (cause ?: return this).getRootCause()
@@ -137,7 +142,8 @@ fun makeIpSpan(ip: InetAddress) = ip.hostAddress.let {
     }
 }
 fun makeMacSpan(mac: String) = if (app.hasTouch) SpannableString(mac).apply {
-    setSpan(CustomTabsUrlSpan("https://macvendors.co/results/$mac"), 0, length, Spannable.SPAN_EXCLUSIVE_EXCLUSIVE)
+    setSpan(CustomTabsUrlSpan("https://maclookup.app/search/result?mac=$mac"), 0, length,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE)
 } else mac
 
 fun NetworkInterface.formatAddresses(macOnly: Boolean = false) = SpannableStringBuilder().apply {
@@ -234,3 +240,15 @@ fun InvocationHandler.callSuper(interfaceClass: Class<*>, proxy: Any, method: Me
 fun globalNetworkRequestBuilder() = NetworkRequest.Builder().apply {
     if (Build.VERSION.SDK_INT >= 31) setIncludeOtherUidNetworks(true)
 }
+
+@OptIn(InternalCoroutinesApi::class)
+suspend fun <T> connectCancellable(url: String, block: suspend (HttpURLConnection) -> T): T {
+    val conn = URL(url).openConnection() as HttpURLConnection
+    val handle = coroutineContext.job.invokeOnCompletion(true) { conn.disconnect() }
+    try {
+        return block(conn)
+    } finally {
+        handle.dispose()
+        conn.disconnect()
+    }
+}