我的代码工具类库

我平时用的工具类

不多说直接贴代码:

Java

Config 保存配置用的类,很常用而且非常有用(因为经常要保存爬虫的登录配置)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;

import java.io.*;
import java.lang.reflect.*;
import java.util.*;
import java.util.stream.Collectors;

// the class to serialized must have no constructor with parameters
public class Config<T extends Config<T>> implements java.io.Serializable {
private T configInstance; //instance of the config class
private String configFormatString; // formatted text of config
private static Gson jsonDeserializer = new Gson(); // deserializer
private String localPath; // path to storage

/**
* just a constructor with non parameter
*/
public Config() {
}

/**
* provide an json text of the config class, and class path(include package path)
* and storage file path, then use reflect to create an instance of config class
* @param configFormatString formatted text
* @param classPath config text path
* @param localPath storage path
* @throws ClassNotFoundException
* @throws NoSuchMethodException
* @throws IllegalAccessException
* @throws InvocationTargetException
* @throws InstantiationException
*/
@SuppressWarnings("unchecked")
public Config(String configFormatString, String classPath, String localPath)
throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
Constructor<T> constructor = (Constructor<T>) Class.forName(classPath).getDeclaredConstructor();
constructor.setAccessible(true);
configInstance = constructor.newInstance();
this.configFormatString = configFormatString;
this.localPath = localPath;
}

/**
* give an configuration class's instance, and a path to storage
* it will auto set the formatted config text
* <PRE>
* var person = new Person(); // configuration class
* person.setAge(17);
* person.setHeight(180);
* person.setWeight(60);
* person.setName("unknown");
* Config configuration = new Config(person, "config.json"); // registry config instance
* </PRE>
* @param configInstance instance of the config class
* @param localPath storage file path
*/
public Config(T configInstance, String localPath) {
this.configInstance = configInstance;
this.localPath = localPath;
this.configFormatString = jsonDeserializer.toJson(configInstance, new TypeToken<T>() {
}.getType());
}

/**
* @see Config#changeFields(Field, Object)
* @param field field name
* @param value field value
* @param <E> type param of field value
*/
public <E> void change(String field, E value) {
var fields = Arrays.stream(configInstance.getClass().getDeclaredFields())
.map(t -> t.toString().substring(t.toString().lastIndexOf('.') + 1))
.collect(Collectors.toList());
if (fields.contains(field)) {
try {
changeFields(configInstance.getClass().getDeclaredField(field), value);
writeToFile();
System.out.println("changed field has saved to file " + localPath);
} catch (ClassCastException e) {
System.out.println("Wrong Changing value's type");
} catch (NoSuchFieldException | IOException ex) {
ex.printStackTrace();
}
}
}

/**
* get <code>configInstance</code>'s all fields as a list,
* go through the list, if found the same field as parameter
* <code>field</code>, then change the old value of the field
* to new value
* @param field to change's field
* @param value changing value
* @param <E> type param of changing value
*/
private <E> void changeFields(Field field, E value) {
var fields = Arrays.asList(configInstance.getClass().getDeclaredFields());
fields.forEach(t -> {
t.setAccessible(true);
if (t.equals(field)) {
try {
t.set(configInstance, value);
} catch (IllegalAccessException ex) {
ex.printStackTrace();
}
}
});
}

/**
* format the config instance as json text
* this method is static , it could be used
* at any config classes without instantiation
* for example:
* <PRE>
* var person = new Person(); // configuration class
* person.setAge(17);
* person.setHeight(180);
* person.setWeight(60);
* person.setName("unknown");
* var text = Config.formatConfigJsonText(person, new TypeToken<Person>(){});
* </PRE>
* @param config configInstance
* @param typeToken get runtime type
* @param <E> type param of config class
* @return formatted config json text
*/
public static <E> String formatConfigJsonText(E config, TypeToken<E> typeToken) {
return serializeConfig(config, typeToken);
}

/**
* deserialize json text as config object, instantiation not required
* @param text json text
* @param typeToken get runtime type
* @param <E> type param of config class
* @return config's instance
*/
public static <E> E deserializeConfig(String text, TypeToken<E> typeToken) {
return jsonDeserializer.fromJson(text, typeToken.getType());
}

/**
* @see Config#deserializeConfig(String, TypeToken) but instantiation required
* @return config's instance
*/
public T deserializeConfig() {
return jsonDeserializer.fromJson(configFormatString, new TypeToken<T>() {
}.getType());
}

/**
* @see Config#serializeConfig(Object, TypeToken) but instantiation required
* @return serialized text
*/
public String serializeConfig() {
return jsonDeserializer.toJson(configInstance, new TypeToken<T>() {
}.getType());
}

/**
* serialize config instance to json text, instantiation not required
* @param instance config instance
* @param typeToken get runtime type
* @param <E> type param of config's class
* @return serialized text
*/
public static <E> String serializeConfig(E instance, TypeToken<E> typeToken) {
return jsonDeserializer.toJson(instance, typeToken.getType());
}

/**
* use {@link java.io.Serializable} to serialize config to file
* @throws IOException ignored
*/
public void serializedToFile() throws IOException {
ObjectOutputStream objectOutputStream = new ObjectOutputStream(new FileOutputStream(new File(localPath)));
objectOutputStream.writeObject(configInstance);
objectOutputStream.close();
}

/**
* use {@link java.io.Serializable} to read the serialized bytecode file
* @return config's instance
* @throws IOException ignored
*/
@SuppressWarnings("unchecked")
public T readFromFile() throws IOException {
T t = null;
try (var objectInputStream = new ObjectInputStream(new FileInputStream(new File(localPath)))) {
t = (T) objectInputStream.readObject();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
return t;
}

/**
* @see Config#serializedToFile()
* @throws IOException ignored
*/
public void writeToFile() throws IOException {
serializedToFile();
}
}

Task 多线程管理类,平时用到多线程下载文件或者获取页面信息会用到

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
package com.dylech30th.library.mylibrary;

import com.dylech30th.function.KeyValuePair;
import com.dylech30th.library.mylibrary.collectionextension.ListExtension;
import com.google.common.collect.BiMap;

import java.util.*;
import java.util.concurrent.*;
import java.util.function.Consumer;

@SuppressWarnings({"unused", "UnnecessaryReturnStatement", "MismatchedQueryAndUpdateOfCollection", "UnusedReturnValue", "SpellCheckingInspection", "WeakerAccess"})
public class Task<T> {

private Runnable runnable;
private BiMap<UUID, Callable<T>> callableMap;
private ListExtension<Callable<T>> taskList;
private ListExtension<FutureTask<T>> futureTaskList;
private ListExtension<Runnable> completableFutureList;
private ExecutorService threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
private int threadCount = 0;
private int timeOut = 2 * 1000;

public Task(ListExtension<Callable<T>> taskList) {
this.taskList = taskList;
threadPool = ForkJoinPool.commonPool();
}

public Task(ListExtension<Callable<T>> taskList, int threadCount) {
this.taskList = taskList;
this.threadCount = threadCount;
if (threadCount == 0) throw new IllegalArgumentException("thread count must over 0");
}

public void runAsyncAll() {
this.completableFutureList.forEach(CompletableFuture::runAsync);
}

public void runAsyncRandom() {
this.completableFutureList.get(new Random().nextInt(completableFutureList.size())).run();
}

public Task(Runnable runnable) {
this.runnable = runnable;
}

public Task() {
this.threadCount = Runtime.getRuntime().availableProcessors();
}

private Consumer<KeyValuePair<UUID, Callable<T>>> call = t -> {
try {
this.callableMap.get(t.getKey()).call();
} catch (Exception e) {
e.printStackTrace();
}
};

private T invoke(Callable<T> callable) throws Exception {
return callable.call();
}

public Task<T> addTask(Callable<T> callable) {
UUID uuid = UUID.randomUUID();
callableMap.put(uuid, callable);
taskList.add(callable);
return this;
}

public Task<T> removeTask(Callable<T> callable) {
taskList.remove(callable);
callableMap.inverse().remove(callable);
callableMap.inverse();
return this;
}

public Task<T> buildBlockingQuene() {
for (Callable<T> task : taskList) {
this.futureTaskList.add(new FutureTask<>(task));
threadPool.submit(task);
}
return this;
}

public List<T> invokeAll() {
ListExtension<T> list = new ListExtension<>();
this.futureTaskList.forEach(t -> {
try {
list.add(t.get(timeOut, TimeUnit.MILLISECONDS));
} catch (InterruptedException | ExecutionException | TimeoutException e) {
return;
}
});
return list;
}

public void wait(UUID uuid, long timeOut) throws InterruptedException {
callableMap.get(uuid).wait(timeOut);
}

public Task<T> autoAllocate() {
this.threadPool = Executors.newFixedThreadPool(callableMap.size());
buildBlockingQuene();
return this;
}

public void executePool() {
this.threadPool.execute(runnable);
}

public void setTimeOut(int timeOut) {
this.timeOut = timeOut;
}

}

StringUtil 字符串工具类,常用于正则匹配,字符串解析,json序列化/反序列化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
package DataTypeExtensions;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;

public class StringUtils {

/**
* deserializer
*/
private static Gson jsonDeserializer = new GsonBuilder().setPrettyPrinting().create();

public static String encodeToUTF8(String text) throws UnsupportedEncodingException {
return URLEncoder.encode(text, "UTF-8");
}

public static String encodeToGBK(String text) throws UnsupportedEncodingException {
return URLEncoder.encode(text, "GBK");
}

public static String decodeUTF8(String text) throws UnsupportedEncodingException {
return URLDecoder.decode(text, "UTF-8");
}

public static String decodeGBK(String text) throws UnsupportedEncodingException {
return URLDecoder.decode(text, "GBK");
}

public static String patternFirst(String source, String patterns, int group) {
var pattern = Pattern.compile(patterns);
var matcher = pattern.matcher(source);
if (matcher.find()) return matcher.group(group);
return "No patterns";
}

public static List<String> patternEach(String source, String patterns, int group) {
var list = new ArrayList<String>();
var pattern = Pattern.compile(patterns);
var matcher = pattern.matcher(source);
while (matcher.find()) {
list.add(matcher.group(group));
}
return list;
}

/**
* replace each text that match trimMaps key-value set
* @param source source string
* @param trimMap key->to replace value->replaced value
* @return trimmed string
*/
public static String trim(String source, Map<String, String> trimMap) {
var str = new AtomicReference<String>();
trimMap.keySet().forEach(t -> str.set(source.replaceAll(t, trimMap.get(t))));
return str.get();
}

/**
* is <code>text</code> made of integer
* @param text to test
* @return is made of integer
*/
public static boolean isInteger(String text) {
return Pattern.compile("^[-+]?[\\d]*$").matcher(text).matches();
}

public static <T> String serializeJson(T obj) {
return jsonDeserializer.toJson(obj, new TypeToken<T>(){}.getType());
}

public static <T> T deserializeJson(String text) {
return jsonDeserializer.fromJson(text, new TypeToken<T>(){}.getType());
}
}

ListUtil 集合工具类,用处很杂,因为Java Collections Api自带的方法着实不够…

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
public class ListExtension<E> extends ArrayList<E> {
private static final Random random = new Random();

public ListExtension() {
super();
}

public ListExtension(int initialCapacity) {
super(initialCapacity);
}

public ListExtension(Collection<? extends E> c) {
super(c);
}

public ListExtension<E> sortBy(Comparator<? super E> c) {
super.sort(c);
return this;
}

public ListExtension<E> removeOn(Predicate<? super E> filter) {
super.removeIf(filter);
return this;
}

/**
* pass in a list and two elements in this list,
* give an sub list of input list from <code>t1</code>
* to <code>t2</code>
* @param list list to cut
* @param t1 begin object
* @param t2 to object
* @return sub list of the original list
*/
public List<E> subListFromElement(List<E> list, E t1, E t2) {
if (!list.contains(t1) || !list.contains(t2))
throw new NoSuchElementException("list does not contains objects t1 or t2");
return list.subList(list.indexOf(t1), list.indexOf(t2));
}

public boolean containsElement(List<E> l1, List<E> l2, E element) {
if (!l1.contains(element)) return false;
return (l1.contains(element) && l2.contains(element));
}

public static <E> boolean containsElement(List<E> l1, List<E> l2) {
for (E e : l1) {
for (E ex : l2) {
if (e.equals(ex))
return true;
}
}
return false;
}

/**
* sort <code>toSort</code> according <code>comparator</code>
* @param toSort list to sort
* @param comparator comparator of the <code>sort()</code>'s param
* @param <T> type parameter of <code>toSort</code>'s element
* @return sorted list
*/
public<T> List<T> sortedList(List<T> toSort, Comparator<? super T> comparator) {
return toSort.stream().sorted(comparator).collect(Collectors.toList());
}

/**
* pick a random value of <code>list</code>
* @param list to pick
* @param <T> type parameter of <code>list</code>'s element
* @return picked element
*/
public static <T> T pickRandom(List<T> list) {
return list.get(random.nextInt(list.size()));
}

/**
* {@link CompareList#compare(Object, Object)}
* @param before before sort
* @param field field name
* @return sorted list of <code>before</code>
*/
public List<E> sortFromFieldValue(List<E> before, String field) {
return before.stream()
.sorted(new ListExtension.CompareList<>(field))
.collect(Collectors.toList());
}

/**
* {@link CompareList#compare(Object, Object)}
* @param before before sort
* @param field field name
* @return sorted stream of <code>before</code>
*/
public Stream<E> sortFromFieldValue(Collection<E> before, String field) {
return before.stream().sorted(new ListExtension.CompareList<>(field));
}

public static <T> boolean isNullOrEmpty(@Nullable T source) {
if (source instanceof String)
return source == null;
else
return source.equals(null);
}
/**
* wrapper class
* @param <T> type parameter
*/
public static class CompareList<T> implements Comparator<T> {

/**
* just a constructor!
* @param field to-compared field's name
*/
CompareList(String field) {
this.field = field;
}

private String field; // to-compared field's name

/**
* get field <code>fieldValue</code>'s value of <code>obj</code>
* @param obj object to get type
* @param fieldValue name of the field to search
* @return value of the field (int/Integer only)
* @throws IllegalAccessException ignored
* @throws NoSuchFieldException ignored
*/
@SuppressWarnings("unchecked")
private int getFieldType(T obj, String fieldValue) throws IllegalAccessException, NoSuchFieldException {
if (obj instanceof Integer || obj instanceof String || obj instanceof Boolean || obj instanceof Double || obj instanceof Float || obj instanceof Character || obj instanceof Byte)
throw new IllegalArgumentException("type parameter cannot be base data type");
Field field = obj.getClass().getDeclaredField(fieldValue);
field.setAccessible(true);
Object value = field.get(obj);
if (!(field.get(obj) instanceof Integer))
try {
return Integer.parseInt((String) value);
} catch (Exception e) {
throw new IllegalArgumentException("selected field value cannot be compared");
}
return (int) value;
}

/**
* override from {@link Comparator<T>}, in order to
* input as {@link Stream#sorted(Comparator)}'s parameter
* @param o1 to compare
* @param o2 compare obj
* @return 0/1 status of compare result
*/
@Override
public int compare(T o1, T o2) {
int value = Integer.MAX_VALUE;
try {
value = -Integer.compare(getFieldType(o1, field), getFieldType(o2, field));
} catch (IllegalAccessException | NoSuchFieldException e) {
e.printStackTrace();
}
return value;
}
}
}

ConnectionExtension网络请求扩展类,没什么好说的,最常用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
package com.dylech30th.library.mylibrary.netutils;

import com.dylech30th.library.mylibrary.Task;
import com.dylech30th.library.mylibrary.baseextension.StringUtil;
import com.dylech30th.library.mylibrary.collectionextension.MapExtension;
import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;
import java.util.function.Consumer;

public class ConnectExtension {
private String url;
private Map<String, String> header;
private static final CookieStore cookieStore = new BasicCookieStore();
private static final ExecutorService threadPool = Executors.newFixedThreadPool(32);
private final CloseableHttpClient httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(requestConfig).build();
private static RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(5000).setConnectionRequestTimeout(5000)
.setSocketTimeout(5000).build();

public static class HttpResponse<T>{
@Getter @Setter
private List<Cookie> cookieList;
@Getter @Setter
private T entity;
}

public ConnectExtension(String url) {
this.url = url;
}

public ConnectExtension(String url, @Nullable Map<String, String> header) {
this.url = url;
this.header = header;
}
public HttpResponse<String> postUseHttpClient(String postData) throws IOException {
HttpResponse<String> httpResponse = new HttpResponse<>();
HttpPost postMethod = new HttpPost(url);
if (header != null)
header.forEach(postMethod::setHeader);
StringEntity entity = new StringEntity(postData, "UTF-8");
entity.setContentEncoding("UTF-8");
postMethod.setEntity(entity);

CloseableHttpResponse response = this.httpClient.execute(postMethod);
httpResponse.setEntity(EntityUtils.toString(response.getEntity(), "UTF-8"));
httpResponse.setCookieList(cookieStore.getCookies());
return httpResponse;
}

public <T> HttpResponse<T> deserializePostEntity(String postData, TypeToken<T> typeToken) throws IOException {
HttpPost postMethod = new HttpPost(url);
HttpResponse<T> httpResponse = new HttpResponse<>();

if (header != null)
header.forEach(postMethod::setHeader);

StringEntity entity = new StringEntity(postData, "UTF-8");

entity.setContentEncoding("UTF-8");
postMethod.setEntity(entity);

String responseEntity = EntityUtils.toString(httpClient.execute(postMethod).getEntity(), "UTF-8");

httpResponse.setCookieList(cookieStore.getCookies());
httpResponse.setEntity(StringUtil.deserializeJson(responseEntity, typeToken));
return httpResponse;
}

public static HttpResponse<String> getUseHttpClient(String url, Map<String, String> header) throws IOException {
HttpResponse<String> httpResponse = new HttpResponse<>();
HttpGet getMethod = new HttpGet(url);
if (header != null) header.forEach(getMethod::setHeader);
CloseableHttpResponse response = HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(requestConfig).build().execute(getMethod);
httpResponse.setEntity(EntityUtils.toString(response.getEntity(), "UTF-8"));
httpResponse.setCookieList(cookieStore.getCookies());
return httpResponse;
}

public HttpResponse<String> getUseHttpClient() throws IOException {
HttpResponse<String> httpResponse = new HttpResponse<>();
HttpGet getMethod = new HttpGet(url);
if (header != null) header.forEach(getMethod::setHeader);
CloseableHttpResponse response = this.httpClient.execute(getMethod);
httpResponse.setEntity(EntityUtils.toString(response.getEntity(), "UTF-8"));
httpResponse.setCookieList(cookieStore.getCookies());
return httpResponse;
}

public static List<HttpResponse<String>> getAsync(Map<String, Map<String, String>> urls) {
List<FutureTask<HttpResponse<String>>> list = new ArrayList<>();
urls.keySet().forEach(t -> list.add(Task.asCallable(t, () -> getUseHttpClient(t, urls.get(t)), threadPool)));
return new Task<>(list).invokeAll();
}

public static Document getWithJsoup(String url, @Nullable Map<String, String> header) {
Connection connect = Jsoup.connect(url);
if (header != null)
header.forEach(connect::header);
Document result = null;
try {
result = connect.ignoreContentType(true).get();
} catch (IOException e) {
e.printStackTrace();
}
return result;
}

public static String sendGet(@Nonnull String url, @Nullable Map<String, String> header) throws IOException {

HttpClient httpClient = HttpClients.createDefault();
HttpGet get = new HttpGet(url);
if (header != null) {
header.forEach(get::setHeader);
}
org.apache.http.HttpResponse res = httpClient.execute(get);
HttpEntity entity = res.getEntity();
return EntityUtils.toString(entity, "UTF-8");
}
}

IOUtils 文件读取写入

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import java.io.*;

public class IOUtils {
@SuppressWarnings("ResultOfMethodCallIgnored")
public static void write(String path, String str) throws IOException {
File file = new File(path);
if(file.exists()) {
file.delete();
file.createNewFile();
} else
file.createNewFile();
try(RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw")) {
randomAccessFile.seek(randomAccessFile.length());
randomAccessFile.write((str + '\n').getBytes());
} catch (IOException e) {
e.printStackTrace();
}
}

public static String read(String path) {
String result;
StringBuilder stringBuilder = new StringBuilder();
try(BufferedReader bufferedReader = new BufferedReader(new FileReader(new File(path)))) {
while ((result = bufferedReader.readLine()) != null)
stringBuilder.append(result);
} catch (IOException e) {
e.printStackTrace();
}
return stringBuilder.toString();
}
}

Kotlin

kotlin工具类 (其实这个东西是我练习kotlin时候移植的java代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
@file:Suppress("UNUSED_EXPRESSION")

package util.extensions

import com.google.gson.GsonBuilder
import com.google.gson.reflect.TypeToken
import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.methods.HttpPost
import org.apache.http.cookie.Cookie
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.BasicCookieStore
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import util.animat.AnimatedGifEncoder
import java.io.File
import java.io.IOException
import java.io.InputStream
import java.net.*
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.*
import java.util.concurrent.Executors
import java.util.regex.Pattern
import java.util.stream.Stream
import java.util.zip.ZipInputStream
import javax.annotation.Nonnull
import javax.annotation.Nullable
import javax.imageio.ImageIO
import kotlin.Comparator
import kotlin.NoSuchElementException
import kotlin.streams.toList

private val stringBuilder = StringBuilder()
private val jsonDeserializer = GsonBuilder().setPrettyPrinting().create()
private val DOWNLOAD_POOL = Executors.newFixedThreadPool(16)
fun <T> T.requireNonNull(): T = Objects.requireNonNull(this)
fun String.encodeToUTF8(): String = URLEncoder.encode(this, "UTF-8")
fun String.encodeToGBK(): String = URLEncoder.encode(this, "GBK")
fun String.decodeUTF8(): String = URLDecoder.decode(this, "UTF-8")
fun String.decodeGBK(): String = URLDecoder.decode(this, "GBK")
fun String.unicodeToString(): String {
this.split("\\\\u").forEach {
val data = Integer.parseInt(it, 16)
stringBuilder.append(data.toChar())
}
return stringBuilder.toString()
}
fun String.patternFirst(patterns: String, group: Int): String {
val ptr = Pattern.compile(patterns)
val mat = ptr.matcher(this)
return if (mat.find()) mat.group(group) else "No Patterns"
}
fun String.patternEach(patterns: String, group: Int): List<String> {
val list = ArrayList<String>()
val pattern = Pattern.compile(patterns)
val matcher = pattern.matcher(this)
while (matcher.find()) list.add(matcher.group(group))
return list
}
infix fun String.trim(trimMap: Map<String, String>): String {
trimMap.forEach { this.replace(it.key, it.value) }
return this
}
fun String.isInt(): Boolean = Pattern.compile("^[-+]?[\\d]*$").matcher(this).matches()
fun <T> T.serializeJson(): String = jsonDeserializer.toJson(this, object: TypeToken<T>(){}.type)
fun <T> String.deserializeJson(typeToken: TypeToken<T>): T = jsonDeserializer.fromJson<T>(this, typeToken.type)
fun <T> String.deserializeJson(clazz: Class<*>): T = jsonDeserializer.fromJson<T>(this, clazz)
fun List<Cookie>.trimToChain(): String {
this.forEach { stringBuilder.append(it.name).append('=').append(it.value).append(';') }
return stringBuilder.toString()
}
fun <E> List<E>.subListFromElement(t1: E, t2: E): List<E> = when(!this.contains(t1) || !this.contains(t2)) {
false -> this.subList(this.indexOf(t1), this.indexOf(t2))
else -> throw NoSuchElementException("list does not contains objects t1 or t2")
}
fun <E> List<E>.containsElement(l2: List<E>, element: E): Boolean = if (!this.contains(element)) false else (this.contains(element) && l2.contains(element))
fun <E> List<E>.containsAll(l2: List<E>, element: E): Boolean {
for (e in this)
for (ex in l2)
if (e!! == ex)
return true
return false
}
fun <T : Comparable<T>, E> List<E>.sortedList(selector: (E) -> T?) =
this.asSequence().sortedWith(compareBy(selector)).toList()
fun <E> List<E>.pickRandom(): E = this[Random().nextInt(this.size)]
fun <T> T.isNull() = when(this) {
is String? -> this.isNullOrEmpty()
else -> this == null
}
class ListExtension<E : Any> {
fun List<E>.sortFromFieldValue(field: String): List<E> = this.stream()
.sorted(ListExtension.ComparableList<E>(field))
.toList()
fun Collection<E>.sortFromFieldValue(field: String): Stream<E> = this.stream()
.sorted(ListExtension.ComparableList<E>(field))
class ComparableList<T : Any>(field: String) : Comparator<T> {
private val field = field

private fun getRuntimeField(obj: T, fieldValue: String): Int {
if (obj is Int || obj is String || obj is Boolean || obj is Double || obj is Float || obj is Char || obj is Byte)
throw IllegalArgumentException("type parameter cannot be base data type")
val field = obj::class.java.getDeclaredField(fieldValue)
val value: Any = field.get(obj)
if (value !is Int)
try {
return Integer.parseInt(value.toString())
} catch (e: Exception) {
throw IllegalArgumentException("selected field value cannot be compared")
}
return value
}
override fun compare(o1: T, o2: T): Int {
var value = Int.MAX_VALUE
try {
value = -Integer.compare(getRuntimeField(o1, field), getRuntimeField(o2, field))
} catch (e: Exception) {
when(e) {
is IllegalAccessException -> e.printStackTrace()
is NoSuchFieldException -> e.printStackTrace()
}
}
return value
}
}
}

@Suppress("UNUSED_EXPRESSION")
class ConnectionExtension (url: String, header: Map<String, String>? = null){
data class HttpResponse<T>(val entity: T, val cookieList: List<Cookie>)
val cookieStore = BasicCookieStore()
private var url: String? = url
private var header: Map<String, String>? = header
private val requestConfig = RequestConfig.custom()
.setConnectTimeout(5000).setConnectionRequestTimeout(5000)
.setSocketTimeout(5000).build()
private val httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(requestConfig).build()

fun requestUseHttpClient(postData: String? = null): HttpResponse<String> {
fun post(): HttpResponse<String> {
val postMethod = HttpPost(url)
when(header != null) {
true -> header!!.forEach { postMethod.setHeader(it.key, it.value) }
}
val entity = StringEntity(postData, "UTF-8")
entity.setContentEncoding("UTF-8")
postMethod.entity = entity

val res = this.httpClient.execute(postMethod)
return HttpResponse(EntityUtils.toString(res.entity, "UTF-8"), cookieStore.cookies)
}
fun get(): HttpResponse<String> {
val getMethod = HttpGet(url)
when(header != null) {
true -> header!!.forEach { getMethod.setHeader(it.key, it.value) }
}
val res = this.httpClient.execute(getMethod)
return HttpResponse(EntityUtils.toString(res.entity, "UTF-8"), cookieStore.cookies)
}
return when(postData != null) {
true -> post()
else -> get()
}
}
inline fun <T> deserializeRequestEntity(postData: String, typeToken: TypeToken<T>): HttpResponse<T> {
val response = requestUseHttpClient(postData)
return HttpResponse(response.entity.deserializeJson(typeToken), cookieStore.cookies)
}
companion object {
fun requestWithJsoup(url: String, header: Map<String, String>? = null): Document? {
val connect = Jsoup.connect(url)
header?.forEach { connect.header(it.key, it.value) }
var result: Document? = null
try {
result = connect.ignoreContentType(true).get()
} catch (e: IOException) {
e.printStackTrace()
}
return result
}
fun sendRequest(@Nonnull url: String, header: Map<String, String>? = null): String {
val clients = HttpClients.createDefault()
val get = HttpGet(url)
header?.forEach { get.setHeader(it.key, it.value) }
val res = clients.execute(get)
return EntityUtils.toString(res.entity, "UTF-8")
}
}
}
@Synchronized private fun jpgToGif(pic: Collection<String>, newPic: String) {
try {
val e = AnimatedGifEncoder()
e.setRepeat(0)
e.start(newPic)
e.setDelay(0)
pic.asSequence().map { File(it) }.map {
try {
return@map ImageIO.read(it)
} catch (ex: IOException) {
ex.printStackTrace()
return@map null
}
}.filter { Objects.nonNull(it) }.forEach { e::addFrame }
e.finish()
} catch (e: Exception) {
println("jpgToGif Failed:")
e.printStackTrace()
}
}
fun trans(files: Array<File>, path: String) {
jpgToGif(files.requireNonNull().asSequence().map { File::toString.call() }.toList(), path)
}
fun URL.readByte(header: Map<String, String>? = null): ByteArray {
val connection = this.openConnection() as HttpURLConnection
header?.forEach(connection::setRequestProperty)
return connection.inputStream.use { it.readBytes() }
}
private fun nioDownload(url: String, path: String, @Nullable header: Map<String, String>? = null) =
File(path).writeBytes(URL(url).readByte(header))
fun download(url: String, path: String, @Nullable header: Map<String, String>? = null) = util.DOWNLOAD_POOL.execute {
try {
nioDownload(url, path, header)
} catch (ignored: IOException) {}
}
private fun setReferer(connection: URLConnection) {
connection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible;MSIE 7.0; Windows 10; Chrome;)")
connection.setRequestProperty("Accept-Encoding", "gzip")
connection.setRequestProperty("Referer", "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=")
connection.setRequestProperty("cookie", "https://www.pixiv.net")
}
fun createConnection(url: URL): HttpURLConnection {
try {
val connection = url.openConnection() as HttpURLConnection
setReferer(connection)
connection.connectTimeout = 5 * 1000
connection.readTimeout = 10 * 1000
connection.connect()
return connection
} catch (e: IOException) {throw e}
}
@Throws(IOException::class)
fun write(path: String, str: String) = File(path).appendText(str)
fun read(path: String): String = File(path).readText()
fun unzip(zipFileName: String, dstPath: Path) {
try {
ZipInputStream(Files.newInputStream(Paths.get(zipFileName))).use {
if (!Files.isDirectory(dstPath)) Files.createDirectories(dstPath)
extract(dstPath, zipInputStream = it)
}
} catch (e: IOException) {e.printStackTrace(); throw e}
}
fun unzip(inputStream: InputStream, dstPath: Path) {
try {
ZipInputStream(inputStream).use {
if (!Files.isDirectory(dstPath)) Files.createDirectories(dstPath)
extract(dstPath, zipInputStream = it)
}
} catch (e: IOException) {e.printStackTrace(); throw e}
}
fun extract(dstPath: Path, zipInputStream: ZipInputStream) {
val entry = zipInputStream.nextEntry
while (entry != null) {
val toPath = dstPath.resolve(entry.name)
if (entry.isDirectory) Files.createDirectory(toPath) else Files.copy(zipInputStream, toPath)
}
}

Cs

Configuration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

using System;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
using Newtonsoft.Json;

namespace Libraries
{
public class Configuration<T> where T : Configuration<T>,
new()
{
private T _instance;

private string _configFormatString;

private string _localPath;

public Configuration(string localPath)
{
_localPath = localPath;
}

public Configuration(T instance, string configFormatString, string localPath)
{
_instance = instance;
_configFormatString = configFormatString;
_localPath = localPath;
}

public Configuration(T instance, string localPath)
{
_instance = instance;
_localPath = localPath;
}

public static string FormatJsonText<E>(E instance)
{
return SerializeConfig(instance);
}

public static string SerializeConfig<E>(E instance)
{
return JsonConvert.SerializeObject(instance);
}

public static E DeserializeConfig<E>(string text)
{
return JsonConvert.DeserializeObject<E>(text);
}

public string SerializeConfig()
{
return _instance.SerializeToJson();
}

public T DeserializeConfig()
{
return _configFormatString.DeserializeJsonText<T>();
}

public void SerializeToFile()
{
using (var fs = new FileStream(_localPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
{
var formatter = new BinaryFormatter();
formatter.Serialize(fs, _instance);

}
}

public void WriteToFile()
{
SerializeToFile();
}

public T DeserializeFromFile()
{
T t;
using (var fs = new FileStream(_localPath, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
{
var formatter = new BinaryFormatter();
t = formatter.Deserialize(fs) as T;
}

return t;
}
}
}

CollectionExtensions

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Libraries
{
public static class CollectionExtension
{

private static readonly Random _random = new Random();

public static T PickRandom<T>(this List<T> source)
{
return source[_random.Next(source.Count)];
}

public static ParallelLoopResult ParallelForeach<T>(this IEnumerable<T> source, Action<T> action)
{
return Parallel.ForEach(source, action);
}

public static bool ContainsElement<T>(this List<T> l1, List<T> l2)
{
foreach (var e in l1)
{
foreach (var ex in l2)
{
if (e.Equals(ex))
return true;
}
}

return false;
}

public static bool ContainsAll<E>(this List<E> l1, List<E> l2)
{
foreach (var e in l2)
{
if (!l1.Contains(e))
{
return false;
}
}

return true;
}

public static List<T>[] SubListPerElement<T>(this List<T> list, int count)
{

var num = Math.Ceiling((double)(list.Count / count));

var reference = new List<T>[(int)num];

for (var i = 0; i < num; i++)
{
var index = count * i;

var subList = list.Skip(index - count).Take(count).ToList();

reference.SetValue(subList, i);
}

return reference;
}

public static string ListToString<T>(this List<T> list)
{
var stringBuilder = new StringBuilder();

foreach (var value in list)
{
if (list.IndexOf(value) != list.Count - 1)
{
stringBuilder.Append('\"').Append(value).Append('\"').Append(',').Append(' ');
}
else
{
stringBuilder.Append('\"').Append(value).Append('\"').Append(' ');
}
}

return $"[{stringBuilder}]";
}

public static bool IsEmpty<T>(this IEnumerable<T> source)
{
return !source.Any();
}

public static IEnumerable<T> RemoveIf<T>(this List<T> source, Predicate<T> predicate)
{
source.RemoveAll(predicate);
return source;
}

public static IEnumerable<T> OrderByFunc<T, TKey>(this IEnumerable<T> source, Func<T, TKey> comparator)
{
return source.OrderBy(comparator);
}

public static IEnumerable<T> DescendOrderByFunc<T, TKey>(this IEnumerable<T> source, Func<T, TKey> comparator)
{
return source.OrderByDescending(comparator);
}
}
}

IOUtils

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using System.Threading;

namespace Libraries
{
public static class IOUtil
{
public static void DownloadFile(this string uri, string path, string fileName,
Dictionary<string, string> header = null)
{
if (!Directory.Exists(path)) Directory.CreateDirectory(path);

header = new Dictionary<string, string>();
var client = new WebClient();

if (!header.IsEmpty())
foreach (var element in header.Keys)
client.Headers.Add(element, header[element]);
try
{
client.DownloadFile(uri, path);
}
catch (Exception)
{
client.DownloadFile(uri.Replace(".jpg", ".png"), path + fileName.Replace(".jpg", ".png"));
}
}

public static void DownloadFileAsync(this string uri, string path, string fileName,
Dictionary<string, string> header = null)
{
ThreadPool.SetMinThreads(16, 16);

ThreadPool.QueueUserWorkItem(t => uri.DownloadFile(path, fileName, header));
}

public static string ReadFile(string path)
{
if (!new FileInfo(path).Exists) throw new FileNotFoundException("File does not exist");

var stringBuilder = new StringBuilder();

using (var streamReader = new StreamReader(path))
{
string str;
while ((str = streamReader.ReadLine()) != null)
{
stringBuilder.Append(str);
}
}

return stringBuilder.ToString();
}

public static void DeleteFiles(string path)
{
if (File.Exists(path))
{
new FileInfo(path).Delete();
return;
}

var directoryInfo = new DirectoryInfo(path);
var fileSystemInfo = directoryInfo.GetFileSystemInfos();
foreach (var systemInfo in fileSystemInfo)
{
if (systemInfo is DirectoryInfo)
{
var directory = new DirectoryInfo(systemInfo.FullName);
directory.Delete(true);
}
else
{
File.Delete(systemInfo.FullName);
}
}
}

public static void Write(this string str, string path)
{
using (var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
{
using (var writer = new StreamWriter(stream))
{
writer.Write(str);
}
}
}

public static void WriteResultToFile<T>(this Dictionary<T, string> dic, string location)
{
foreach (var element in dic.Keys)
{
dic[element].JsonFormatter().Write(location);
}
}
}
}

StringUtil

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using Newtonsoft.Json;

namespace Libraries
{
public static class StringUtil
{
private static readonly StringBuilder StringBuilder = new StringBuilder();
public static string PatternFirst(this string source, string pattern, int group)
{
var match = Regex.Match(source, pattern);
return match.Groups[@group].ToString();
}

public static bool IsNullOrEmpty(this string str) => String.IsNullOrEmpty(str);

public static bool IsNullOrWhiteSpace(this string str) => String.IsNullOrWhiteSpace(str);

public static Dictionary<T, string> StringFormatter<T>(this List<T> list, List<string> invokeList)
{
var dic = new Dictionary<T, string>();

foreach (var value in list)
{
var type = value.GetType();

var stringBuilder = new StringBuilder();

foreach (var element in invokeList)
{
var v = type.GetProperty(element)?.GetValue(value, null).ToString();

if (list.IndexOf(value) == 0)
{
stringBuilder.Append('{').Append($"\"{element}\":").Append($"\"{v}\"").Append(',');
}
else if (list.IndexOf(value) == list.Count - 1)
{
stringBuilder.Append($"\"{element}\":").Append($"\"{v}\"").Append('}');
}
else
{
stringBuilder.Append($"\"{element}\":").Append($"\"{v}\"").Append(',');
}
}

dic.Add(value, stringBuilder.ToString());
}

return dic;
}

public static List<string> PatternEach(this string source, string pattern, int group)
{
var returnList = new List<string>();

var matches = Regex.Matches(source, pattern);
foreach (Match match in matches) returnList.Add(match.Groups[@group].ToString());

return returnList;
}

public static T DeserializeJsonText<T>(this string jsonText)
{
return JsonConvert.DeserializeObject<T>(jsonText);
}

public static string SerializeToJson<T>(this T source)
{
return JsonConvert.SerializeObject(source);
}


public static string JsonFormatter(this string source)
{
var serializer = new JsonSerializer();

var jsonReader = new JsonTextReader(new StringReader(source));

var instance = serializer.Deserialize(jsonReader);

if (!instance.IsNull())
{
var writer = new JsonTextWriter(new StringWriter())
{
Formatting = Formatting.Indented,
Indentation = 4,
IndentChar = ' '
};

serializer.Serialize(writer, instance);

return writer.ToString();
}

return source;
}

public static string EncodeTo(this string str, Encoding charset)
{
return HttpUtility.UrlEncode(str, charset);
}

public static string DecodeAs(this string str, Encoding charset)
{
return HttpUtility.UrlDecode(str, charset);
}
}
}

WebUtils

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;

namespace NetEaseBox
{
public class WebUtilities
{

public class HttpResponse<T>
{
public T Entity { get; set; }

public List<Cookie> CookieContainer { get; set; }
}

private readonly Dictionary<string, string> _header;
private readonly string _url;

public WebUtilities(string url, Dictionary<string, string> header)
{
_url = url;
_header = header;
}

public WebUtilities(string url)
{
_url = url;
_header = new Dictionary<string, string>();
}

public void DownloadToLocal(string filePath)
{
var client = new WebClient();
if (_header != null)
{
foreach (var keyValuePair in _header)
{
client.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
}
client.DownloadFile(_url, filePath);
}

public byte[] SendRequest(string postData = null)
{


var client = new WebClient();
if (_header != null)
{
foreach (var keyValuePair in _header)
{
client.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
}

return postData != null
? client.UploadData(_url, postData.GetBytes(Encoding.UTF8))
: client.DownloadData(_url);
}

private void SetHeader(ref HttpWebRequest request)
{
foreach (var t in _header)
{
switch (t.Key)
{
case "Host":
request.Host = t.Value;
break;
case "Referer":
request.Referer = t.Value;
break;
case "Accept":
request.Accept = t.Value;
break;
case "User-Agent":
request.UserAgent = t.Value;
break;
case "Content-Length":
request.ContentLength = long.Parse(t.Value);
break;
case "Content-Type":
request.ContentType = t.Value;
break;
case "Keep-Alive":
request.KeepAlive = bool.Parse(t.Value);
break;
case "Connection":
request.Connection = t.Value;
break;
case "Range":
request.AddRange(int.Parse(t.Value.Split(' ')[0]), int.Parse(t.Value.Split(' ')[1]));
break;
default:
request.Headers[t.Key] = t.Value;
break;
}
}
}

public HttpResponse<string> SendRequestByHttpWebRequest(string url, string postData = null)
{
HttpWebRequest request;
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
request = WebRequest.Create(url) as HttpWebRequest;
request.ProtocolVersion = HttpVersion.Version10;
}
else
{
request = WebRequest.Create(url) as HttpWebRequest;
}

var cookieContainer = new CookieContainer();

request.CookieContainer = new CookieContainer();

SetHeader(ref request);

if (postData != null)
{
var bytes = postData.GetBytes(Encoding.UTF8);
request.Method = "POST";

using (var st = request.GetRequestStream())
{
st.Write(bytes, 0, bytes.Length);
}
}

request.CookieContainer = cookieContainer;

return GetResponse(request);
}
private static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain,
SslPolicyErrors errors) => true;

public void DownloadToLocalByHttpWebRequest(string savePath)
{
var request = GetHttpWebRequest(_url);

SetHeader(ref request);
using (var response = request.GetResponse() as HttpWebResponse)
{
using (var responseStream = response.GetResponseStream())
{
using (var stream = new FileStream(savePath, FileMode.Create))
{
var buffer = new byte[1024];
var size = 0;
while ((size = responseStream.Read(buffer, 0, buffer.Length)) > 0)
{
stream.Write(buffer, 0, size);
}

stream.Close();
}
}
}
}

public static string TryDownload(string url, Dictionary<string, string> header = null)
{
var request = GetHttpWebRequest(url);
if (header != null)
{
foreach (var t in header)
{
switch (t.Key)
{
case "Host":
request.Host = t.Value;
break;
case "Referer":
request.Referer = t.Value;
break;
case "Accept":
request.Accept = t.Value;
break;
case "User-Agent":
request.UserAgent = t.Value;
break;
case "Content-Length":
request.ContentLength = long.Parse(t.Value);
break;
case "Content-Type":
request.ContentType = t.Value;
break;
case "Keep-Alive":
request.KeepAlive = bool.Parse(t.Value);
break;
case "Connection":
request.Connection = t.Value;
break;
case "Range":
break;
default:
request.Headers[t.Key] = t.Value;
break;
}
}
}
request.AddRange(0, 0);
var response = request.GetResponse();
return response.Headers["Content-Range"];

}

private static HttpWebRequest GetHttpWebRequest(string url)
{
HttpWebRequest request;
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
request = WebRequest.Create(url) as HttpWebRequest;
request.ProtocolVersion = HttpVersion.Version10;
}
else
{
request = WebRequest.Create(url) as HttpWebRequest;
}
request.AutomaticDecompression = DecompressionMethods.GZip;
request.Timeout = 8000;
request.Timeout = 120000;
request.KeepAlive = false;
return request;
}

private HttpResponse<string> GetResponse(HttpWebRequest request)
{
var response = new HttpResponse<string>();
var cookies = request.CookieContainer.GetCookies(request.RequestUri);

using (var res = (HttpWebResponse)request.GetResponse())
{
var stream = res.GetResponseStream();
stream.ReadTimeout = 20000;

var result = new StreamReader(stream, Encoding.UTF8).ReadToEnd();

var cookieList = new List<Cookie>().AddAll(cookies);

response.Entity = result;
response.CookieContainer = cookieList;
}

return response;
}

public T DeserializeRequestEntity<T>()
{
var result = SendRequestByHttpWebRequest(_url);

return result.Entity.DeserializeJsonText<T>();
}

public HttpResponse<T> DeserializeRequestEntity<T>(string postData = null)
{
var response = postData == null
? SendRequestByHttpWebRequest(_url)
: SendRequestByHttpWebRequest(_url, postData);

var entity = response.Entity;

return new HttpResponse<T>
{
CookieContainer = response.CookieContainer,
Entity = entity.DeserializeJsonText<T>()
};
}
}
}

EncodingUtils

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
using System.IO;
using System.IO.Compression;
// ReSharper disable IdentifierTypo


namespace BiliBiliDanmakuCrawler
{
public static class EncodingUtil
{
public static byte[] DecompressDeflateBytes(this byte[] data)
{
var ms = new MemoryStream(data)
{
Position = 0
};
var outMs = new MemoryStream();
using (var deflateStream = new DeflateStream(ms, CompressionMode.Decompress, true))
{
var buffer = new byte[1024];
int length;
while ((length = deflateStream.Read(buffer, 0, buffer.Length)) > 0)
{
outMs.Write(buffer, 0, length);
}
}

return outMs.ToArray();
}

public static byte[] DecompressGzipBytes(this byte[] data)
{
var ms = new MemoryStream(data);
var outBuffer = new MemoryStream();
using (var gzipStream = new GZipStream(ms, CompressionMode.Decompress))
{

var buffer = new byte[1024];
int length;
while ((length = gzipStream.Read(buffer, 0, buffer.Length)) > 0)
{
outBuffer.Write(buffer, 0, length);
}
}

return outBuffer.ToArray();
}
}
}
0%