oxen-io
/
session-android
mirror of https://github.com/oxen-io/session-android.git
1
0
Fork 0
Code Issues 1 Projects Releases Wiki Activity

improve link preview to support more sites

This commit is contained in:
Ryan ZHAO 2020-11-18 15:17:38 +11:00
parent 55da7056dc
commit d6c5892a15
3 changed files with 183 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java
View File

@ -10,7 +10,6 @@ import android.text.Html;
import android.text.TextUtils;
import com.bumptech.glide.load.engine.DiskCacheStrategy;
import com.bumptech.glide.request.FutureTarget;
import com.google.android.gms.common.util.IOUtils;
import org.thoughtcrime.securesms.ApplicationContext;
@ -18,7 +17,6 @@ import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.attachments.UriAttachment;
import org.thoughtcrime.securesms.database.AttachmentDatabase;
import org.thoughtcrime.securesms.dependencies.InjectableType;
import org.thoughtcrime.securesms.giph.model.ChunkedImageUrl;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.mms.GlideApp;
import org.thoughtcrime.securesms.net.CallRequestController;
@ -37,6 +35,7 @@ import org.whispersystems.libsignal.util.guava.Optional;
import org.whispersystems.signalservice.api.SignalServiceMessageReceiver;
import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifest;
import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifest.StickerInfo;
import org.thoughtcrime.securesms.linkpreview.LinkPreviewUtil.OpenGraph;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -116,7 +115,8 @@ public class LinkPreviewRepository implements InjectableType {
}
private @NonNull RequestController fetchMetadata(@NonNull String url, Callback<Metadata> callback) {
Call call = client.newCall(new Request.Builder().url(url).cacheControl(NO_CACHE).build());
Call call = client.newCall(new Request.Builder().url(url).removeHeader("User-Agent").addHeader("User-Agent",
"WhatsApp").cacheControl(NO_CACHE).build());
call.enqueue(new okhttp3.Callback() {
@Override
@ -138,8 +138,9 @@ public class LinkPreviewRepository implements InjectableType {
}
String body = response.body().string();
Optional<String> title = getProperty(body, "title");
Optional<String> imageUrl = getProperty(body, "image");
OpenGraph openGraph = LinkPreviewUtil.parseOpenGraphFields(body);
Optional<String> title = openGraph.getTitle();
Optional<String> imageUrl = openGraph.getImageUrl();
if (imageUrl.isPresent() && !LinkPreviewUtil.isValidMediaUrl(imageUrl.get())) {
Log.i(TAG, "Image URL was invalid or for a non-whitelisted domain. Skipping.");
@ -161,6 +162,8 @@ public class LinkPreviewRepository implements InjectableType {
try {
Response response = call.execute();
if (!response.isSuccessful() || response.body() == null) {
controller.cancel();
callback.onComplete(Optional.absent());
return;
}
@ -184,24 +187,6 @@ public class LinkPreviewRepository implements InjectableType {
return controller;
}
private @NonNull Optional<String> getProperty(@NonNull String searchText, @NonNull String property) {
Pattern pattern = Pattern.compile("<\\s*meta\\s+property\\s*=\\s*\"\\s*og:" + property + "\\s*\"\\s+[^>]*content\\s*=\\s*\"(.*?)\"[^>]*/?\\s*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
Matcher matcher = pattern.matcher(searchText);
if (matcher.find()) {
String text = Html.fromHtml(matcher.group(1)).toString();
if (!TextUtils.isEmpty(text)) { return Optional.of(text); }
}
pattern = Pattern.compile("<\\s*" + property + "[^>]*>(.*?)<\\s*/" + property + "[^>]*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
matcher = pattern.matcher(searchText);
if (matcher.find()) {
String text = Html.fromHtml(matcher.group(1)).toString();
if (!TextUtils.isEmpty(text)) { return Optional.of(text); }
}
return Optional.absent();
}
private RequestController fetchStickerPackLinkPreview(@NonNull Context context,
@NonNull String packUrl,
@NonNull Callback<Optional<LinkPreview>> callback)

143
src/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java
View File

@ -2,6 +2,9 @@ package org.thoughtcrime.securesms.linkpreview;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import android.annotation.SuppressLint;
import android.text.Html;
import android.text.SpannableString;
import android.text.TextUtils;
import android.text.style.URLSpan;
@ -10,9 +13,14 @@ import android.text.util.Linkify;
import com.annimon.stream.Stream;
import org.thoughtcrime.securesms.stickers.StickerUrl;
import org.thoughtcrime.securesms.util.DateUtils;
import org.thoughtcrime.securesms.util.Util;
import org.whispersystems.libsignal.util.guava.Optional;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -20,10 +28,15 @@ import okhttp3.HttpUrl;
public final class LinkPreviewUtil {
private static final Pattern DOMAIN_PATTERN = Pattern.compile("^(https?://)?([^/]+).*$");
private static final Pattern ALL_ASCII_PATTERN = Pattern.compile("^[\\x00-\\x7F]*$");
private static final Pattern ALL_NON_ASCII_PATTERN = Pattern.compile("^[^\\x00-\\x7F]*$");
private static final Pattern STICKER_URL_PATTERN = Pattern.compile("^.*#pack_id=(.*)&pack_key=(.*)$");
private static final Pattern DOMAIN_PATTERN = Pattern.compile("^(https?://)?([^/]+).*$");
private static final Pattern ALL_ASCII_PATTERN = Pattern.compile("^[\\x00-\\x7F]*$");
private static final Pattern ALL_NON_ASCII_PATTERN = Pattern.compile("^[^\\x00-\\x7F]*$");
private static final Pattern OPEN_GRAPH_TAG_PATTERN = Pattern.compile("<\\s*meta[^>]*property\\s*=\\s*\"\\s*og:([^\"]+)\"[^>]*/?\\s*>");
private static final Pattern ARTICLE_TAG_PATTERN = Pattern.compile("<\\s*meta[^>]*property\\s*=\\s*\"\\s*article:([^\"]+)\"[^>]*/?\\s*>");
private static final Pattern OPEN_GRAPH_CONTENT_PATTERN = Pattern.compile("content\\s*=\\s*\"([^\"]*)\"");
private static final Pattern TITLE_PATTERN = Pattern.compile("<\\s*title[^>]*>(.*)<\\s*/title[^>]*>");
private static final Pattern FAVICON_PATTERN = Pattern.compile("<\\s*link[^>]*rel\\s*=\\s*\".*icon.*\"[^>]*>");
private static final Pattern FAVICON_HREF_PATTERN = Pattern.compile("href\\s*=\\s*\"([^\"]*)\"");
/**
* @return All whitelisted URLs in the source text.
@ -82,4 +95,126 @@ public final class LinkPreviewUtil {
return false;
}
}
public static @NonNull OpenGraph parseOpenGraphFields(@Nullable String html) {
return parseOpenGraphFields(html, text -> Html.fromHtml(text).toString());
}
static @NonNull OpenGraph parseOpenGraphFields(@Nullable String html, @NonNull HtmlDecoder htmlDecoder) {
if (html == null) {
return new OpenGraph(Collections.emptyMap(), null, null);
}
Map<String, String> openGraphTags = new HashMap<>();
Matcher openGraphMatcher = OPEN_GRAPH_TAG_PATTERN.matcher(html);
while (openGraphMatcher.find()) {
String tag = openGraphMatcher.group();
String property = openGraphMatcher.groupCount() > 0 ? openGraphMatcher.group(1) : null;
if (property != null) {
Matcher contentMatcher = OPEN_GRAPH_CONTENT_PATTERN.matcher(tag);
if (contentMatcher.find() && contentMatcher.groupCount() > 0) {
String content = htmlDecoder.fromEncoded(contentMatcher.group(1));
openGraphTags.put(property.toLowerCase(), content);
}
}
}
Matcher articleMatcher = ARTICLE_TAG_PATTERN.matcher(html);
while (articleMatcher.find()) {
String tag = articleMatcher.group();
String property = articleMatcher.groupCount() > 0 ? articleMatcher.group(1) : null;
if (property != null) {
Matcher contentMatcher = OPEN_GRAPH_CONTENT_PATTERN.matcher(tag);
if (contentMatcher.find() && contentMatcher.groupCount() > 0) {
String content = htmlDecoder.fromEncoded(contentMatcher.group(1));
openGraphTags.put(property.toLowerCase(), content);
}
}
}
String htmlTitle = "";
String faviconUrl = "";
Matcher titleMatcher = TITLE_PATTERN.matcher(html);
if (titleMatcher.find() && titleMatcher.groupCount() > 0) {
htmlTitle = htmlDecoder.fromEncoded(titleMatcher.group(1));
}
Matcher faviconMatcher = FAVICON_PATTERN.matcher(html);
if (faviconMatcher.find()) {
Matcher faviconHrefMatcher = FAVICON_HREF_PATTERN.matcher(faviconMatcher.group());
if (faviconHrefMatcher.find() && faviconHrefMatcher.groupCount() > 0) {
faviconUrl = faviconHrefMatcher.group(1);
}
}
return new OpenGraph(openGraphTags, htmlTitle, faviconUrl);
}
private static @Nullable String parseTopLevelDomain(@NonNull String domain) {
int periodIndex = domain.lastIndexOf(".");
if (periodIndex >= 0 && periodIndex < domain.length() - 1) {
return domain.substring(periodIndex + 1);
} else {
return null;
}
}
public static final class OpenGraph {
private final Map<String, String> values;
private final @Nullable String htmlTitle;
private final @Nullable String faviconUrl;
private static final String KEY_TITLE = "title";
private static final String KEY_DESCRIPTION_URL = "description";
private static final String KEY_IMAGE_URL = "image";
private static final String KEY_PUBLISHED_TIME_1 = "published_time";
private static final String KEY_PUBLISHED_TIME_2 = "article:published_time";
private static final String KEY_MODIFIED_TIME_1 = "modified_time";
private static final String KEY_MODIFIED_TIME_2 = "article:modified_time";
public OpenGraph(@NonNull Map<String, String> values, @Nullable String htmlTitle, @Nullable String faviconUrl) {
this.values = values;
this.htmlTitle = htmlTitle;
this.faviconUrl = faviconUrl;
}
public @NonNull Optional<String> getTitle() {
return Optional.of(Util.getFirstNonEmpty(values.get(KEY_TITLE), htmlTitle));
}
public @NonNull Optional<String> getImageUrl() {
return Optional.of(Util.getFirstNonEmpty(values.get(KEY_IMAGE_URL), faviconUrl));
}
@SuppressLint("ObsoleteSdkInt")
public long getDate() {
return Stream.of(values.get(KEY_PUBLISHED_TIME_1),
values.get(KEY_PUBLISHED_TIME_2),
values.get(KEY_MODIFIED_TIME_1),
values.get(KEY_MODIFIED_TIME_2))
.map(DateUtils::parseIso8601)
.filter(time -> time > 0)
.findFirst()
.orElse(0L);
}
public @NonNull
Optional<String> getDescription() {
return Optional.of(values.get(KEY_DESCRIPTION_URL));
}
}
public interface HtmlDecoder {
@NonNull String fromEncoded(@NonNull String html);
}
}

36
src/org/thoughtcrime/securesms/util/DateUtils.java
View File

@ -16,10 +16,17 @@
*/
package org.thoughtcrime.securesms.util;
import android.annotation.SuppressLint;
import android.content.Context;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import android.os.Build;
import android.text.format.DateFormat;
import org.thoughtcrime.securesms.logging.Log;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
@ -142,4 +149,33 @@ public class DateUtils extends android.text.format.DateUtils {
private static String getLocalizedPattern(String template, Locale locale) {
return DateFormat.getBestDateTimePattern(locale, template);
}
/**
* e.g. 2020-09-04T19:17:51Z
* https://www.iso.org/iso-8601-date-and-time-format.html
*
* Note: SDK_INT == 0 check needed to pass unit tests due to JVM date parser differences.
*
* @return The timestamp if able to be parsed, otherwise -1.
*/
@SuppressLint("ObsoleteSdkInt")
public static long parseIso8601(@Nullable String date) {
SimpleDateFormat format;
if (Build.VERSION.SDK_INT == 0 || Build.VERSION.SDK_INT >= 24) {
format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX", Locale.getDefault());
} else {
format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.getDefault());
}
if (date.isEmpty()) {
return -1;
}
try {
return format.parse(date).getTime();
} catch (ParseException e) {
Log.w(TAG, "Failed to parse date.", e);
return -1;
}
}
}